diff --git a/.travis.yml b/.travis.yml index af5052682f..c6468f78f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,8 @@ env: ## Turn off tor's sandbox in chutney, until we fix sandbox errors that are ## triggered by Ubuntu Xenial and Bionic. See #32722. - CHUTNEY_TOR_SANDBOX="0" + ## The default target for chutney jobs + - CHUTNEY_MAKE="test-network-all" matrix: ## This matrix entry is required, but it doesn't actually create any jobs - @@ -41,34 +43,48 @@ env: matrix: ## include creates builds with gcc, linux, unless we override those defaults include: - ## We run basic tests on macOS + ## We run chutney on macOS, because macOS Travis has IPv6 + ## But we only run the IPv6 chutney tests, to speed up the job + - env: CHUTNEY_MAKE="test-network-ipv6" CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes" + os: osx + + ## We also run basic tests on macOS - compiler: clang os: osx ## Turn off some newer features, turn on clang's -Wtypedef-redefinition env: C_DIALECT_OPTIONS="-std=gnu99" - ## We check NSS - ## Use -std=gnu99 to turn off some newer features, and maybe turn on some - ## extra gcc warnings? - - env: NSS_OPTIONS="--enable-nss" C_DIALECT_OPTIONS="-std=gnu99" + ## We run chutney on Linux, because it's faster than chutney on macOS ## Chutney is a fast job, clang is slower on Linux, so we do Chutney clang - env: CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes" compiler: clang - ## We check asciidoc with distcheck, to make sure we remove doc products - ## We use Linux clang, because there are no other Linux clang jobs - - env: DISTCHECK="yes" ASCIIDOC_OPTIONS="" SKIP_MAKE_CHECK="yes" - compiler: clang - ## We include a single coverage build with the best options for coverage - - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS="" TOR_TEST_RNG_SEED="636f766572616765" - ## We run rust on Linux, because it's faster than rust on macOS - ## We check rust offline - - env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true + ## We check asciidoc with distcheck, to make sure we remove doc products - env: DISTCHECK="yes" ASCIIDOC_OPTIONS="" SKIP_MAKE_CHECK="yes" + ## We check disable module relay - env: MODULES_OPTIONS="--disable-module-relay" ## We check disable module dirauth - env: MODULES_OPTIONS="--disable-module-dirauth" + + ## We run rust on Linux, because it's faster than rust on macOS + ## We check rust offline + - env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true + + ## We check NSS + ## Use -std=gnu99 to turn off some newer features, and maybe turn on some + ## extra gcc warnings? + - env: NSS_OPTIONS="--enable-nss" C_DIALECT_OPTIONS="-std=gnu99" + + ## We include a single coverage build with the best options for coverage + - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS="" TOR_TEST_RNG_SEED="636f766572616765" + + ## We clone our stem repo and run `make test-stem` + - env: TEST_STEM="yes" SKIP_MAKE_CHECK="yes" + + ## We run `make doxygen` without `make check`. + - env: SKIP_MAKE_CHECK="yes" DOXYGEN="yes" + ## macOS builds are very slow, and we have a limited number of ## concurrent macOS jobs. We're not actively developing Rust, so it is ## the lowest priority. @@ -76,13 +92,6 @@ matrix: #- env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode" # compiler: clang # os: osx - ## We run chutney on macOS, because macOS Travis has IPv6 - - env: CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes" - os: osx - ## We clone our stem repo and run `make test-stem` - - env: TEST_STEM="yes" SKIP_MAKE_CHECK="yes" - ## We run `make doxygen` without `make check`. - - env: SKIP_MAKE_CHECK="yes" DOXYGEN="yes" ## Allow the build to report success (with non-required sub-builds ## continuing to run) if all required sub-builds have succeeded. @@ -94,11 +103,17 @@ matrix: ## macOS rust and chutney are very slow, so we let the build finish before ## they are done. We'd like to fast finish, but still eventually show ## any failures in the build status. But Travis doesn't have that ability. - - env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode" - compiler: clang - os: osx - - env: CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes" - os: osx + + ## Since this job is disabled, there's not much point having an exception + ## for it + #- env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode" + # compiler: clang + # os: osx + + ## Since we're actively developing IPv6, we want to require the IPv6 + ## chutney tests + #- env: CHUTNEY_MAKE="test-network-ipv6" CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes" + # os: osx ## (Linux only) Use a recent Linux image (Ubuntu Bionic) dist: bionic @@ -230,7 +245,7 @@ script: ## We run `make check` because that's what https://jenkins.torproject.org does. - if [[ "$SKIP_MAKE_CHECK" == "" ]]; then make check; fi - if [[ "$DISTCHECK" != "" ]]; then make distcheck DISTCHECK_CONFIGURE_FLAGS="$CONFIGURE_FLAGS"; fi - - if [[ "$CHUTNEY" != "" ]]; then make test-network-all; fi + - if [[ "$CHUTNEY" != "" ]]; then make "$CHUTNEY_MAKE"; fi ## Diagnostic for bug 29437: kill stem if it hangs for 9.5 minutes ## Travis will kill the job after 10 minutes with no output - if [[ "$TEST_STEM" != "" ]]; then make src/app/tor; timelimit -p -t 540 -s USR1 -T 30 -S ABRT python3 "$STEM_SOURCE_DIR"/run_tests.py --tor src/app/tor --integ --test control.controller --test control.base_controller --test process --log TRACE --log-file stem.log; fi diff --git a/Makefile.am b/Makefile.am index ac61a990fc..45dab1e6b4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -275,60 +275,175 @@ check-local: \ shellcheck \ check-cocci +# test-network requires a copy of Chutney in $CHUTNEY_PATH. +# Chutney can be cloned from https://git.torproject.org/chutney.git . +.PHONY: need-chutney-path need-chutney-path: @if test ! -d "$$CHUTNEY_PATH"; then \ echo '$$CHUTNEY_PATH was not set.'; \ - if test -d $(top_srcdir)/../chutney -a -x $(top_srcdir)/../chutney/chutney; then \ - echo "Assuming test-network.sh will find" $(top_srcdir)/../chutney; \ + if test -d "$(top_srcdir)/../chutney" && \ + test -x "$(top_srcdir)/../chutney/chutney"; then \ + echo "Assuming test-network.sh will find" \ + "$(top_srcdir)/../chutney"; \ else \ echo; \ - echo "To run these tests, git clone https://git.torproject.org/chutney.git ; export CHUTNEY_PATH=\`pwd\`/chutney"; \ + echo "To run these tests," \ + "git clone" \ + "https://git.torproject.org/chutney.git ;" \ + "export CHUTNEY_PATH=\`pwd\`/chutney"; \ exit 1; \ fi \ fi -# Note that test-network requires a copy of Chutney in $CHUTNEY_PATH. -# Chutney can be cloned from https://git.torproject.org/chutney.git . -test-network: need-chutney-path $(TESTING_TOR_BINARY) src/tools/tor-gencert - $(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_FLAGS) +# Run some basic tests using automake's test-driver +.PHONY: test-network +# Make prints all these recursive invocations by default, but those extra lines +# break up the chutney flavor statuses +.SILENT: test-network +test-network: + @$(MAKE) test-network-mkdir + @$(MAKE) test-network-clean + @$(MAKE) test-network-ipv4-impl \ + ipv4_flavors="$(TEST_CHUTNEY_FLAVOR_QUICK)" + @$(MAKE) test-network-ipv6-impl \ + ipv6_flavors="$(TEST_CHUTNEY_FLAVOR_QUICK_IPV6)" + @$(MAKE) test-network-results # Run all available tests using automake's test-driver +.PHONY: test-network-all +.SILENT: test-network-all +test-network-all: + @$(MAKE) test-network-mkdir + @$(MAKE) test-network-clean + @$(MAKE) test-network-ipv4-impl \ + ipv4_flavors="$(TEST_CHUTNEY_FLAVORS)" + @$(MAKE) test-network-mixed-impl \ + mixed_flavors="$(TEST_CHUTNEY_FLAVORS_MIXED)" + @$(MAKE) test-network-ipv6-impl \ + ipv6_flavors="$(TEST_CHUTNEY_FLAVORS_IPV6)" + @$(MAKE) test-network-results + +# Run IPv4 and mixed tests using automake's test-driver +.PHONY: test-network-ipv4 +.SILENT: test-network-ipv4 +test-network-ipv4: + @$(MAKE) test-network-mkdir + @$(MAKE) test-network-clean + @$(MAKE) test-network-ipv4-impl ipv4_flavors="$(TEST_CHUTNEY_FLAVORS)" + @$(MAKE) test-network-mixed-impl \ + mixed_flavors="$(TEST_CHUTNEY_FLAVORS_MIXED)" + @$(MAKE) test-network-results + +# Run IPv6 tests using automake's test-driver +.PHONY: test-network-ipv6 +.SILENT: test-network-ipv6 +test-network-ipv6: + @$(MAKE) test-network-mkdir + @$(MAKE) test-network-clean + @$(MAKE) test-network-ipv6-impl \ + ipv6_flavors="$(TEST_CHUTNEY_FLAVORS_IPV6)" + @$(MAKE) test-network-results + +# Make the test network log directory, if it does not exist +.PHONY: test-network-mkdir +.SILENT: test-network-mkdir +test-network-mkdir: + @mkdir -p "$(TEST_NETWORK_ALL_LOG_DIR)" + +# Clean the test network log directory +.PHONY: test-network-clean +# We don't silence clean: we want to see the rm command +# We need to remove all matching files, so we can't quote the glob part of the +# rm arguments +test-network-clean: + rm -f "$(TEST_NETWORK_ALL_LOG_DIR)"/*.log \ + "$(TEST_NETWORK_ALL_LOG_DIR)"/*.trs + +# Run the IPv4-only test flavors +# - $(ipv4_flavors) contains the tests to run +.PHONY: test-network-ipv4-impl +.SILENT: test-network-ipv4-impl +test-network-ipv4-impl: + @$(MAKE) test-network-run flavors="$(ipv4_flavors)" skip_flavors="" + +# Run the mixed tests in $(mixed_flavors), if a tor-stable binary is available +# - only run mixed tests if we have a tor-stable binary +# - $(mixed_flavors) contains the tests to run (or skip) +.PHONY: test-network-mixed-impl +.SILENT: test-network-mixed-impl +test-network-mixed-impl: + @flavors=""; \ + if command -v tor-stable >/dev/null 2>&1; then \ + echo "tor-stable found, running mixed flavors:" \ + "$(mixed_flavors)."; \ + flavors="$$flavors $(mixed_flavors)"; \ + else \ + echo "tor-stable not found, skipping mixed flavors:" \ + "$(mixed_flavors)."; \ + skip_flavors="$$skip_flavors $(mixed_flavors)"; \ + fi; \ + $(MAKE) test-network-run flavors="$$flavors" \ + skip_flavors="$$skip_flavors" + +# Run the IPv6 tests in $(ipv6_flavors), if IPv6 is available # - only run IPv6 tests if we can ping6 or ping -6 ::1 (localhost) # we try the syntax for BSD ping6, Linux ping6, and Linux ping -6, # because they're incompatible # - some IPv6 tests may fail without an IPv6 DNS server # (see #16971 and #17011) -# - only run mixed tests if we have a tor-stable binary +# - $(ipv6_flavors) contains the tests to run (or skip) +.PHONY: test-network-ipv6-impl +.SILENT: test-network-ipv6-impl +test-network-ipv6-impl: + @flavors=""; \ + if ping6 -q -c 1 -o ::1 >/dev/null 2>&1 || \ + ping6 -q -c 1 -W 1 ::1 >/dev/null 2>&1 || \ + ping -6 -c 1 -W 1 ::1 >/dev/null 2>&1; then \ + echo "ping6 ::1 or ping ::1 succeeded, running IPv6" \ + "flavors: $(ipv6_flavors)."; \ + flavors="$$flavors $(ipv6_flavors)"; \ + else \ + echo "ping6 ::1 and ping ::1 failed, skipping IPv6 flavors:" \ + "$(ipv6_flavors)."; \ + skip_flavors="$$skip_flavors $(ipv6_flavors)"; \ + fi; \ + $(MAKE) test-network-run flavors="$$flavors" \ + skip_flavors="$$skip_flavors" + +# Run tests using automake's test-driver +# - $(flavors) contains the tests to run +# - $(skip_flavors) contains the tests to skip +.PHONY: test-network-run +.SILENT: test-network-run +# We need the word splitting in the "for" lines, so we can't quote +# $(skip_flavors) or $(flavors) +test-network-run: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/tor-gencert + @for f in $(skip_flavors); do \ + echo "SKIP: $$f"; \ + done; \ + for f in $(flavors); do \ + $(SHELL) "$(top_srcdir)/test-driver" --test-name "$$f" \ + --log-file "$(TEST_NETWORK_ALL_LOG_DIR)/$$f.log" \ + --trs-file "$(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs" \ + $(TEST_NETWORK_ALL_DRIVER_FLAGS) \ + "$(top_srcdir)/src/test/test-network.sh" \ + --flavor "$$f" $(TEST_NETWORK_FLAGS); \ + "$(top_srcdir)/src/test/test-network.sh" \ + $(TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS); \ + done + +# Print the results from automake's test-driver # - show tor warnings on the console after each network run # (otherwise, warnings go to the logs, and people don't see them unless # there is a network failure) -test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/tor-gencert - mkdir -p $(TEST_NETWORK_ALL_LOG_DIR) - rm -f $(TEST_NETWORK_ALL_LOG_DIR)/*.log $(TEST_NETWORK_ALL_LOG_DIR)/*.trs - @flavors="$(TEST_CHUTNEY_FLAVORS)"; \ - if ping6 -q -c 1 -o ::1 >/dev/null 2>&1 || ping6 -q -c 1 -W 1 ::1 >/dev/null 2>&1 || ping -6 -c 1 -W 1 ::1 >/dev/null 2>&1; then \ - echo "ping6 ::1 or ping ::1 succeeded, running IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \ - flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \ - else \ - echo "ping6 ::1 and ping ::1 failed, skipping IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \ - skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \ - fi; \ - if command -v tor-stable >/dev/null 2>&1; then \ - echo "tor-stable found, running mixed flavors: $(TEST_CHUTNEY_FLAVORS_MIXED)."; \ - flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_MIXED)"; \ - else \ - echo "tor-stable not found, skipping mixed flavors: $(TEST_CHUTNEY_FLAVORS_MIXED)."; \ - skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_MIXED)"; \ - fi; \ - for f in $$skip_flavors; do \ - echo "SKIP: $$f"; \ - done; \ - for f in $$flavors; do \ - $(SHELL) $(top_srcdir)/test-driver --test-name $$f --log-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.log --trs-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs $(TEST_NETWORK_ALL_DRIVER_FLAGS) $(top_srcdir)/src/test/test-network.sh --flavor $$f $(TEST_NETWORK_FLAGS); \ - $(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS); \ - done; \ - echo "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."; \ - ! grep -q FAIL $(TEST_NETWORK_ALL_LOG_DIR)/*.trs +.PHONY: test-network-results +.SILENT: test-network-results +# We need to grep all matching files, so we can't quote the glob part of the +# grep arguments +test-network-results: + @echo \ + "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)." + @! grep -q FAIL "$(TEST_NETWORK_ALL_LOG_DIR)"/*.trs need-stem-path: @if test ! -d "$$STEM_SOURCE_DIR"; then \ diff --git a/changes/ticket28208 b/changes/ticket28208 new file mode 100644 index 0000000000..8818aad1d5 --- /dev/null +++ b/changes/ticket28208 @@ -0,0 +1,4 @@ + o Testing: + - Use the "bridges+hs-v23" chutney network flavor in "make test-network". + This test requires a recent version of chutney (mid-February 2020). + Closes ticket 28208. diff --git a/changes/ticket33194 b/changes/ticket33194 new file mode 100644 index 0000000000..b87e55348e --- /dev/null +++ b/changes/ticket33194 @@ -0,0 +1,4 @@ + o Testing: + - Remove a redundant distcheck job. Closes ticket 33194. + - Sort the Travis jobs in order of speed. Putting the slowest jobs first + takes full advantage of Travis job concurrency. Closes ticket 33194. diff --git a/changes/ticket33195 b/changes/ticket33195 new file mode 100644 index 0000000000..11abd4816e --- /dev/null +++ b/changes/ticket33195 @@ -0,0 +1,4 @@ + o Testing: + - Stop allowing the Chutney IPv6 Travis job to fail. This job was + previously configured to fast_finish (which requires allow_failure), to + speed up the build. Closes ticket 33195. diff --git a/changes/ticket33280 b/changes/ticket33280 new file mode 100644 index 0000000000..b90c3086ea --- /dev/null +++ b/changes/ticket33280 @@ -0,0 +1,4 @@ + o Testing: + - Add test-network-ipv4 and test-network-ipv6 jobs to the Makefile. + These jobs run the IPv4-only and dual-stack chutney flavours from + test-network-all. Closes ticket 33280. diff --git a/changes/ticket33300 b/changes/ticket33300 new file mode 100644 index 0000000000..9b0bdce372 --- /dev/null +++ b/changes/ticket33300 @@ -0,0 +1,3 @@ + o Testing: + - Add a basic IPv6 test to "make test-network". This test only runs when + the local machine has an IPv6 stack. Closes ticket 33300. diff --git a/changes/ticket33303 b/changes/ticket33303 new file mode 100644 index 0000000000..b7ac7b5067 --- /dev/null +++ b/changes/ticket33303 @@ -0,0 +1,4 @@ + o Testing: + - Run the test-network-ipv6 Makefile target in the Travis CI IPv6 chutney + job. This job runs on macOS, so it's a bit slow. + Closes ticket 33303. diff --git a/src/test/include.am b/src/test/include.am index 90e50752ce..fdd1195fdf 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -61,11 +61,21 @@ TESTS += src/test/test src/test/test-slow src/test/test-memwipe \ $(TESTSCRIPTS) # These flavors are run using automake's test-driver and test-network.sh + +# run a quick test or two +# this test only uses IPv4 +TEST_CHUTNEY_FLAVOR_QUICK = bridges+hs-v23 +# only run if we can ping6 ::1 (localhost) +TEST_CHUTNEY_FLAVOR_QUICK_IPV6 = client-ipv6-only-md + +# run a basic set of tests, which only use IPv4 TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-v2-min hs-v3-min \ single-onion-v23 + # only run if we can ping6 ::1 (localhost) TEST_CHUTNEY_FLAVORS_IPV6 = bridges+ipv6-min ipv6-exit-min hs-v23-ipv6-md \ single-onion-v23-ipv6-md + # only run if we can find a stable (or simply another) version of tor TEST_CHUTNEY_FLAVORS_MIXED = mixed+hs-v2