From 2c43b7ea471c819c8e726eefccc39a469f6b9ae6 Mon Sep 17 00:00:00 2001 From: Michael Wallner Date: Tue, 27 Oct 2020 09:22:08 +0100 Subject: [PATCH] testing: errors --- test/lib/Cluster.cpp | 19 +-- test/lib/Cluster.hpp | 5 +- test/lib/Connection.cpp | 1 - test/tests/memcached/errors.cpp | 62 +++++++++ tests/failure.cc | 237 -------------------------------- 5 files changed, 74 insertions(+), 250 deletions(-) delete mode 100644 tests/failure.cc diff --git a/test/lib/Cluster.cpp b/test/lib/Cluster.cpp index e1375bfd..e32f0fc3 100644 --- a/test/lib/Cluster.cpp +++ b/test/lib/Cluster.cpp @@ -13,7 +13,16 @@ Cluster::Cluster(Server serv, uint16_t cnt) if (!count) { count = 1; } - reset(); + for (int i = 0; i < count; ++i) { + cluster.push_back(proto); + } +} + +Cluster::Cluster(vector servers) +: count{servers.size()} +, cluster{move(servers)} +{ + } Cluster::~Cluster() { @@ -25,14 +34,6 @@ const vector &Cluster::getServers() const { return cluster; } -void Cluster::reset() { - pids.clear(); - cluster.clear(); - for (int i = 0; i < count; ++i) { - cluster.push_back(proto); - } -} - bool Cluster::start() { bool started = true; diff --git a/test/lib/Cluster.hpp b/test/lib/Cluster.hpp index 1cc00ac8..199d2c74 100644 --- a/test/lib/Cluster.hpp +++ b/test/lib/Cluster.hpp @@ -21,7 +21,7 @@ class Cluster { public: explicit Cluster(Server serv, uint16_t cnt = 0); - + explicit Cluster(vector servers); ~Cluster(); Cluster(const Cluster &c) = delete; @@ -43,13 +43,12 @@ public: bool start(); void stop(); - void reset(); bool isStopped(); bool isListening(); void wait(); private: - uint16_t count; + size_t count; Server proto; vector cluster; map pids; diff --git a/test/lib/Connection.cpp b/test/lib/Connection.cpp index b6df553b..0530d38b 100644 --- a/test/lib/Connection.cpp +++ b/test/lib/Connection.cpp @@ -193,6 +193,5 @@ string Connection::error(const initializer_list &args) { ss << arg; } - cerr << ss.str() << endl; return ss.str(); } diff --git a/test/tests/memcached/errors.cpp b/test/tests/memcached/errors.cpp index 16ac61a7..75ba019a 100644 --- a/test/tests/memcached/errors.cpp +++ b/test/tests/memcached/errors.cpp @@ -1,4 +1,6 @@ #include "test/lib/common.hpp" +#include "test/lib/MemcachedCluster.hpp" +#include "test/lib/Retry.hpp" TEST_CASE("memcached_errors") { SECTION("NO_SERVERS") { @@ -13,4 +15,64 @@ TEST_CASE("memcached_errors") { REQUIRE(MEMCACHED_NO_SERVERS == rc); REQUIRE(MEMCACHED_NO_SERVERS == memcached_mget(*memc, &key, &len, 1)); } + + SECTION("dead servers") { + MemcachedCluster test{Cluster{vector{Server{MEMCACHED_BINARY, {"-p", random_port_string("-p")}}}}}; + auto memc = &test.memc; + + REQUIRE_SUCCESS(memcached_set(memc, S("foo"), nullptr, 0, 0, 0)); + memcached_quit(memc); + + test.cluster.stop(); + Retry cluster_is_stopped{[&cluster = test.cluster]{ + return cluster.isStopped(); + }}; + REQUIRE(cluster_is_stopped()); + + SECTION("TEMPORARILY_DISABLED") { + REQUIRE_SUCCESS(memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_RETRY_TIMEOUT, 3)); + REQUIRE_RC(MEMCACHED_CONNECTION_FAILURE, memcached_set(memc, S("foo"), nullptr, 0, 0, 0)); + REQUIRE_RC(MEMCACHED_SERVER_TEMPORARILY_DISABLED, memcached_set(memc, S("foo"), nullptr, 0, 0, 0)); + } + + SECTION("recovers from TEMPORARILY_DISABLED") { + REQUIRE_SUCCESS(memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_RETRY_TIMEOUT, 1)); + REQUIRE_RC(MEMCACHED_CONNECTION_FAILURE, memcached_set(memc, S("foo"), nullptr, 0, 0, 0)); + REQUIRE_RC(MEMCACHED_SERVER_TEMPORARILY_DISABLED, memcached_set(memc, S("foo"), nullptr, 0, 0, 0)); + + REQUIRE(test.cluster.start()); + Retry cluster_is_listening{[&cluster = test.cluster] { + return cluster.isListening(); + }}; + REQUIRE(cluster_is_listening()); + + Retry recovers{[memc]{ + return MEMCACHED_SUCCESS == memcached_set(memc, S("foo"), nullptr, 0, 0, 0); + }, 50, 100ms}; + REQUIRE(recovers()); + } + + SECTION("MARKED_DEAD") { + SECTION("immediately") { + REQUIRE_SUCCESS(memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_REMOVE_FAILED_SERVERS, true)); + REQUIRE_SUCCESS(memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_SERVER_FAILURE_LIMIT, 1)); + + REQUIRE_RC(MEMCACHED_CONNECTION_FAILURE, memcached_set(memc, S("foo"), nullptr, 0, 0, 0)); + REQUIRE_RC(MEMCACHED_SERVER_MARKED_DEAD, memcached_set(memc, S("foo"), nullptr, 0, 0, 0)); + } + SECTION("with retry") { + REQUIRE_SUCCESS(memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_REMOVE_FAILED_SERVERS, true)); + REQUIRE_SUCCESS(memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_SERVER_FAILURE_LIMIT, 2)); + REQUIRE_SUCCESS(memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_RETRY_TIMEOUT, 1)); + + REQUIRE_RC(MEMCACHED_CONNECTION_FAILURE, memcached_set(memc, S("foo"), nullptr, 0, 0, 0)); + REQUIRE_RC(MEMCACHED_SERVER_TEMPORARILY_DISABLED, memcached_set(memc, S("foo"), nullptr, 0, 0, 0)); + + Retry server_is_marked_dead{[memc] { + return MEMCACHED_SERVER_MARKED_DEAD == memcached_set(memc, S("foo"), nullptr, 0, 0, 0); + },50, 100ms}; + REQUIRE(server_is_marked_dead()); + } + } + } } diff --git a/tests/failure.cc b/tests/failure.cc deleted file mode 100644 index facaa7d5..00000000 --- a/tests/failure.cc +++ /dev/null @@ -1,237 +0,0 @@ -/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: - * - * Libmemcached library - * - * Copyright (C) 2011 Data Differential, http://datadifferential.com/ - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * * The names of its contributors may not be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "mem_config.h" - -/* - C++ interface test -*/ -#include "libmemcached-1.0/memcached.hpp" -#include "libmemcached/server_instance.h" -#include "libtest/test.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -using namespace std; -using namespace memcache; -using namespace libtest; - -libtest::Framework *global_framework= NULL; - -static test_return_t shutdown_servers(memcached_st *memc) -{ - test_compare(memcached_server_count(memc), 1U); - - // Disable a single server, just the first - test_true(global_framework->servers().shutdown(0)); - - return TEST_SUCCESS; -} - -static test_return_t add_shutdown_servers(memcached_st *memc) -{ - while (memcached_server_count(memc) < 2) - { - const char *argv[2]= { "add_shutdown_server", NULL }; - in_port_t port = libtest::default_port(); - test_true(global_framework->servers().start_socket_server("memcached", port, argv)); - test_compare(MEMCACHED_SUCCESS, memcached_server_add(memc, "localhost", port)); - } - - // Disable a single server, just the first - test_true(global_framework->servers().shutdown(0)); - - return TEST_SUCCESS; -} - -static test_return_t restart_servers(memcached_st *) -{ - // Restart the servers - global_framework->servers().restart(); - - return TEST_SUCCESS; -} - -#include "libmemcached/instance.hpp" -static test_return_t cull_TEST(memcached_st *memc) -{ - shutdown_servers(memc); - - uint32_t count= memcached_server_count(memc); - - // Do not do this in your code, it is not supported. - memc->servers[0].options.is_dead= true; - memc->state.is_time_for_rebuild= true; - - uint32_t new_count= memcached_server_count(memc); - test_compare(count, new_count); - - return restart_servers(memc); -} - -static test_return_t MEMCACHED_SERVER_TEMPORARILY_DISABLED_TEST(memcached_st *memc) -{ - shutdown_servers(memc); - - test_compare(MEMCACHED_SUCCESS, memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_RETRY_TIMEOUT, 30)); - test_compare_got(MEMCACHED_CONNECTION_FAILURE, - memcached_set(memc, - test_literal_param("foo"), - NULL, 0, time_t(0), uint32_t(0)), - memcached_last_error_message(memc)); - - /* - Setting server_failure_counter==0 should not influence the timeout that we set above, - since we check the timeout that is created by the failure before we check how many times - a server has failed. - */ - test_compare(MEMCACHED_SERVER_TEMPORARILY_DISABLED, - memcached_set(memc, test_literal_param("foo"), NULL, 0, time_t(0), uint32_t(0))); - - return restart_servers(memc); -} - -static test_return_t MEMCACHED_SERVER_TEMPORARILY_DISABLED_to_success_TEST(memcached_st *memc) -{ - shutdown_servers(memc); - - test_compare_got(MEMCACHED_CONNECTION_FAILURE, - memcached_set(memc, - test_literal_param("foo"), - NULL, 0, time_t(0), uint32_t(0)), - memcached_last_error_message(memc)); - - /* - Setting server_failure_counter==0 should not influence the timeout that we set above, - since we check the timeout that is created by the failure before we check how many times - a server has failed. - */ - test_compare(MEMCACHED_SERVER_TEMPORARILY_DISABLED, - memcached_set(memc, test_literal_param("foo"), NULL, 0, time_t(0), uint32_t(0))); - - global_framework->servers().restart(); - - int limit= 10; - memcached_return_t ret; - do { - libtest::dream(1, 0); - ret= memcached_set(memc, test_literal_param("foo"), NULL, 0, time_t(0), uint32_t(0)); - } while (ret == MEMCACHED_SERVER_TEMPORARILY_DISABLED and --limit); - - test_true(limit); - - test_compare_got(MEMCACHED_SUCCESS, ret, memcached_last_error_message(memc)); - - return restart_servers(memc); -} - -static test_return_t MEMCACHED_SERVER_MARKED_DEAD_TEST(memcached_st *memc) -{ - add_shutdown_servers(memc); - - test_compare(MEMCACHED_SUCCESS, memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_SERVER_FAILURE_LIMIT, 2)); - test_compare(MEMCACHED_SUCCESS, memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_RETRY_TIMEOUT, 1)); - test_compare(MEMCACHED_SUCCESS, memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_AUTO_EJECT_HOSTS, true)); - - memcached_return_t ret; - do { - ret= memcached_set(memc, - test_literal_param("foo"), - NULL, 0, time_t(0), uint32_t(0)); - } while (ret == MEMCACHED_SUCCESS or ret == MEMCACHED_CONNECTION_FAILURE); - test_compare(MEMCACHED_SERVER_TEMPORARILY_DISABLED, ret); - - int limit= 10; - do { - libtest::dream(1, 0); - ret= memcached_set(memc, test_literal_param("foo"), NULL, 0, time_t(0), uint32_t(0)); - } while ((ret == MEMCACHED_SERVER_TEMPORARILY_DISABLED or ret == MEMCACHED_SUCCESS) and --limit); - - test_true(limit); - - test_compare_got(MEMCACHED_SERVER_MARKED_DEAD, ret, memcached_last_error_message(memc)); - - return restart_servers(memc); -} - -test_st cull_TESTS[] ={ - { "cull servers", true, (test_callback_fn*)cull_TEST }, - { 0, 0, 0 } -}; - -test_st server_temporarily_disabled_TESTS[] ={ - { "memcached_set(MEMCACHED_SERVER_TEMPORARILY_DISABLED -> MEMCACHED_SUCCESS)", true, (test_callback_fn*)MEMCACHED_SERVER_TEMPORARILY_DISABLED_to_success_TEST }, - { "memcached_set(MEMCACHED_SERVER_TEMPORARILY_DISABLED)", true, (test_callback_fn*)MEMCACHED_SERVER_TEMPORARILY_DISABLED_TEST }, - { 0, 0, 0 } -}; - -test_st server_permanently_disabled_TESTS[] ={ - { "memcached_set(MEMCACHED_SERVER_MARKED_DEAD)", true, (test_callback_fn*)MEMCACHED_SERVER_MARKED_DEAD_TEST }, - { 0, 0, 0 } -}; - -collection_st collection[] ={ - { "cull", NULL, NULL, cull_TESTS }, - { "server failed", NULL, NULL, server_temporarily_disabled_TESTS }, - { "server eject", NULL, NULL, server_permanently_disabled_TESTS }, - { 0, 0, 0, 0 } -}; - -#include "tests/libmemcached_world.h" - -void get_world(libtest::Framework* world) -{ - world->servers().set_servers_to_run(1); - - world->collections(collection); - - world->create((test_callback_create_fn*)world_create); - world->destroy((test_callback_destroy_fn*)world_destroy); - - world->set_runner(new LibmemcachedRunner); - - global_framework= world; -} -- 2.30.2