From: Michael Wallner Date: Thu, 13 Aug 2020 11:54:32 +0000 (+0200) Subject: Merge branch 'docs' into v1.0 X-Git-Tag: 1.1.0-beta1~250 X-Git-Url: https://git.m6w6.name/?a=commitdiff_plain;h=911c5ad81ebd8ff9a72cfbb5bc1e2a7b49ee3f92;hp=492497f0aa08f6f3229922c38df21e4dbde7ad02;p=awesomized%2Flibmemcached Merge branch 'docs' into v1.0 --- diff --git a/.gitignore b/.gitignore index dffd1bf9..ffe575ca 100644 --- a/.gitignore +++ b/.gitignore @@ -42,10 +42,11 @@ src/bin/memslap src/bin/memstat src/bin/memtouch cmake-build-*/ +ccmake-*/ example/memcached_light example/t/memcached_light tags venv/ /infer-out/ -/docs/gh-pages/source/ +/docs/gh-pages/pages/ /docs/gh-pages/build/ diff --git a/CMake/CheckDebug.cmake b/CMake/CheckDebug.cmake index 749e188b..cc91da97 100644 --- a/CMake/CheckDebug.cmake +++ b/CMake/CheckDebug.cmake @@ -4,7 +4,7 @@ include(CheckCXXCompilerFlag) function(check_debug) if(CMAKE_BUILD_TYPE STREQUAL "Debug") add_compile_options(-O1) - add_compile_definitions(DEBUG=1) + add_definitions(-DDEBUG=1) foreach(FLAG IN ITEMS -fno-inline -fno-omit-frame-pointer @@ -59,6 +59,6 @@ function(check_debug) endif() endif() else() - add_compile_definitions(DEBUG=0) + add_definitions(-DDEBUG=0) endif() endfunction() diff --git a/CMake/CheckDependency.cmake b/CMake/CheckDependency.cmake index e1bdd5b9..3cb57535 100644 --- a/CMake/CheckDependency.cmake +++ b/CMake/CheckDependency.cmake @@ -10,8 +10,8 @@ function(check_dependency NAME LIB HEADER) pkg_check_modules(${NAME} ${LIB}${ARGN}) endif() set(${NAME} ${${NAME}_FOUND} PARENT_SCOPE) - set(${NAME}_LIBRARIES ${${NAME}_LIBRARIES} PARENT_SCOPE) - set(${NAME}_INCLUDEDIR ${${NAME}_INCLUDEDIR}} PARENT_SCOPE) + set(${NAME}_LIBRARIES ${${NAME}_LDFLAGS} PARENT_SCOPE) + set(${NAME}_INCLUDEDIR ${${NAME}_INCLUDEDIR} PARENT_SCOPE) safe_string(${LIB} LIB_CONST) set(HAVE_${NAME} ${${NAME}_FOUND} PARENT_SCOPE) diff --git a/CMake/_Include.cmake b/CMake/_Include.cmake index 1c5b89ff..0f25ab48 100644 --- a/CMake/_Include.cmake +++ b/CMake/_Include.cmake @@ -1,5 +1,3 @@ -set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMake) - set(THREADS_PREFER_PTHREAD_FLAG ON) # globals @@ -50,13 +48,13 @@ if(ENABLE_SASL) endif() ## hashes -if(ENABLE_FNV64_HASH) +if(ENABLE_HASH_FNV64) set(HAVE_FNV64_HASH 1) endif() -if(ENABLE_MURMUR_HASH) +if(ENABLE_HASH_MURMUR) set(HAVE_MURMUR_HASH 1) endif() -if(ENABLE_HSIEH_HASH) +if(ENABLE_HASH_HSIEH) set(HAVE_HSIEH_HASH 1) endif() diff --git a/CMakeConfig.txt b/CMakeConfig.txt index 3acae1cb..41b76324 100644 --- a/CMakeConfig.txt +++ b/CMakeConfig.txt @@ -3,55 +3,52 @@ set(CMAKE_BUILD_TYPE Release set(CMAKE_INSTALL_PREFIX /usr/local CACHE PATH "install prefix") -# make test - -set(BUILD_TESTING ON - CACHE STRING "whether to enable build of the test suite") +option(BUILD_TESTING "whether to enable build of the test suite" + OFF) +option(BUILD_DOCSONLY "build *only* documentation" + OFF) +option(BUILD_DOCS "build documentation" + ${BUILD_DOCSONLY}) +option(BUILD_DOCS_HTML "build HTML docs" + ${BUILD_DOCS}) +option(BUILD_DOCS_MAN "build manpages" + ${BUILD_DOCS}) +option(BUILD_DOCS_MANGZ "gzip manpages" + ${BUILD_DOCS_MAN}) + +option(ENABLE_DTRACE "enable dtrace support" + OFF) +option(ENABLE_HASH_FNV64 "enable fnv64 hash support" + ON) +option(ENABLE_HASH_MURMUR "enable murmur hash support" + ON) +option(ENABLE_HASH_HSIEH "enable hsieh hash support" + OFF) +option(ENABLE_MEMASLAP "enable memaslap client" + ON) +option(ENABLE_SASL "enable SASL support" + OFF) set(ENABLE_SANITIZERS "" CACHE STRING "sanitizers to enable (e.g. address undefined ...)") -set(MEMCACHED_BINARY "/usr/bin/memcached" - CACHE FILEPATH "memcached binary") - -# sasl - -set(ENABLE_SASL OFF - CACHE BOOL "enable SASL support") - -# hashes - -set(ENABLE_FNV64_HASH ON - CACHE BOOL "enable fnv64 hash support") -set(ENABLE_MURMUR_HASH ON - CACHE BOOL "enable murmur hash support") -set(ENABLE_HSIEH_HASH OFF - CACHE BOOL "enable hsieh hash support") - -# memaslap, needs libevent and stdatomic -set(ENABLE_MEMASLAP ON - CACHE BOOL "enable memaslap client") - -# dtrace - -set(ENABLE_DTRACE OFF - CACHE BOOL "enable dtrace support") - -# docs / sphinx - -set(GZIP_MAN ON - CACHE BOOL "gzip manpages") - -set(SPHINX_OPTIONS "" - CACHE STRING "additional sphinx-build command line options") -set(SPHINX_THEME "sphinx_rtd_theme" - CACHE STRING "sphinx HTML theme") -set(SPHINX_THEME_OPTIONS "'collapse_navigation':False, 'navigation_depth':2, 'titles_only':False, 'includehidden':False" - CACHE STRING "sphinx HTML theme options") -set(SPHINX_EXTENSIONS "" - CACHE STRING "comma separated list of quoted sphinx extensions") -set(SPHINX_CONF_APPEND "" - CACHE STRING "append verbatim code to sphinx' conf.py") +if(BUILD_TESTING) + set(MEMCACHED_BINARY "/usr/bin/memcached" + CACHE FILEPATH "memcached binary") +endif() + +if(BUILD_DOCS) + set(SPHINX_OPTIONS "" + CACHE STRING "additional sphinx-build command line options") + set(SPHINX_THEME "sphinx_rtd_theme" + CACHE STRING "sphinx HTML theme") + set(SPHINX_THEME_OPTIONS "" + CACHE STRING "sphinx HTML theme options") + set(SPHINX_EXTENSIONS "" + CACHE STRING "comma separated list of quoted sphinx extensions") + set(SPHINX_CONF_APPEND "" + CACHE STRING "append verbatim code to sphinx' conf.py") +endif() # legacy @@ -61,5 +58,8 @@ set(HAVE_GCC_BUILTIN_ATOMIC 1) # modules +set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMake) -include(CMake/_Include.cmake) +if(NOT BUILD_DOCSONLY) + include(CMake/_Include.cmake) +endif() diff --git a/CMakeLists.txt b/CMakeLists.txt index 7cd6745f..2db4ab3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,40 +17,47 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) include(CMakeConfig.txt) -foreach(INCLUDE IN ITEMS ${CMAKE_BINARY_DIR} .) - include_directories( - ${INCLUDE}/ - ${INCLUDE}/include - ${INCLUDE}/src +if(NOT BUILD_DOCSONLY) + foreach(INCLUDE IN ITEMS ${CMAKE_BINARY_DIR} .) + include_directories( + ${INCLUDE}/ + ${INCLUDE}/include + ${INCLUDE}/src + ) + endforeach() + + set(AUTOHEADER_FILE mem_config.h) + + set(CLIENTS + memcapable + memcat + memcp + memdump + memerror + memexist + memflush + memparse + memping + memrm + memslap + memstat + memtouch ) -endforeach() - -set(AUTOHEADER_FILE mem_config.h) - -set(CLIENTS - memcapable - memcat - memcp - memdump - memerror - memexist - memflush - memparse - memping - memrm - memslap - memstat - memtouch - ) -add_subdirectory(include) -add_subdirectory(src) -add_subdirectory(docs) -add_subdirectory(support) + add_subdirectory(include) + add_subdirectory(src) + add_subdirectory(support) + + if(BUILD_TESTING) + add_subdirectory(tests) + endif() -if(BUILD_TESTING) - add_subdirectory(tests) + # keep last + configure_file(src/mem_config.h.in ${AUTOHEADER_FILE} @ONLY) endif() -# keep last -configure_file(src/mem_config.h.in ${AUTOHEADER_FILE} @ONLY) +if(BUILD_DOCS OR BUILD_DOCSONLY) + add_subdirectory(docs) +endif() + + diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 07709dd0..784fadb7 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -34,50 +34,56 @@ else() "${SPHINX_BUILD_DIR}/conf.py" @ONLY) - add_custom_target(html - ${SPHINX_EXECUTABLE} - -q -b html - -c "${SPHINX_BUILD_DIR}" - -d "${SPHINX_CACHE_DIR}" - ${SPHINX_OPTIONS} - "${SPHINX_SOURCE_DIR}" - "${SPHINX_HTML_DIR}" - BYPRODUCTS ${SPHINX_HTML_DIR} - ) - add_custom_target(man ALL - ${SPHINX_EXECUTABLE} - -q -b man - -c "${SPHINX_BUILD_DIR}" - -d "${SPHINX_CACHE_DIR}" - ${SPHINX_OPTIONS} - "${SPHINX_SOURCE_DIR}" - "${SPHINX_MAN_DIR}" - BYPRODUCTS ${SPHINX_MAN_DIR} - ) - - set(MAN_EXT "") - if(GZIP_MAN) - find_program(PIGZ pigz) - if(PIGZ) - set(GZIP ${PIGZ}) - else() - find_package(UnixCommands) - endif() - if(GZIP) - set(MAN_EXT ".gz") - add_custom_target(man_gz ALL - ${GZIP} -rkf ${SPHINX_MAN_DIR} - DEPENDS man - ) + if(BUILD_DOCS_HTML) + add_custom_target(html + ${SPHINX_EXECUTABLE} + -q -b html + -c "${SPHINX_BUILD_DIR}" + -d "${SPHINX_CACHE_DIR}" + ${SPHINX_OPTIONS} + "${SPHINX_SOURCE_DIR}" + "${SPHINX_HTML_DIR}" + BYPRODUCTS ${SPHINX_HTML_DIR} + ) + endif() + + if(BUILD_DOCS_MAN) + add_custom_target(man ALL + ${SPHINX_EXECUTABLE} + -q -b man + -c "${SPHINX_BUILD_DIR}" + -d "${SPHINX_CACHE_DIR}" + ${SPHINX_OPTIONS} + "${SPHINX_SOURCE_DIR}" + "${SPHINX_MAN_DIR}" + BYPRODUCTS ${SPHINX_MAN_DIR} + ) + + set(MAN_EXT "") + if(BUILD_DOCS_MANGZ) + find_program(PIGZ pigz) + if(PIGZ) + set(GZIP ${PIGZ}) + else() + find_package(UnixCommands) + endif() + if(GZIP) + set(MAN_EXT ".gz") + add_custom_target(man_gz ALL + ${GZIP} -kf ${SPHINX_MAN_DIR}/*.{1,3} + DEPENDS man + ) + endif() endif() + + install(DIRECTORY ${SPHINX_MAN_DIR}/ + DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 + FILES_MATCHING PATTERN *.1${MAN_EXT} + ) + install(DIRECTORY ${SPHINX_MAN_DIR}/ + DESTINATION ${CMAKE_INSTALL_MANDIR}/man3 + FILES_MATCHING PATTERN *.3${MAN_EXT} + ) endif() - install(DIRECTORY ${SPHINX_MAN_DIR}/ - DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 - FILES_MATCHING PATTERN *.1${MAN_EXT} - ) - install(DIRECTORY ${SPHINX_MAN_DIR}/ - DESTINATION ${CMAKE_INSTALL_MANDIR}/man3 - FILES_MATCHING PATTERN *.3${MAN_EXT} - ) endif() diff --git a/docs/gh-pages/publish.sh b/docs/gh-pages/publish.sh index b87f8ef5..5b43fa24 100755 --- a/docs/gh-pages/publish.sh +++ b/docs/gh-pages/publish.sh @@ -3,22 +3,22 @@ set -eu cd "$(dirname $0)" -if test -d source/.git +if test -d pages/.git then - cd source + cd pages git pull -r cd .. else - git clone -b gh-pages github.com:m6w6/libmemcached source + git clone -b gh-pages github.com:m6w6/libmemcached pages fi mkdir -p build cd build -cmake ../.. +cmake -DBUILD_DOCSONLY=true -DBUILD_DOCS_HTML=true ../../.. make html -rsync -va --delete --exclude=.git/ docs/html/ ../source/ +rsync -va --delete --exclude=.git/ docs/html/ ../pages/ -cd ../source +cd ../pages touch .nojekyll git add -A git ci -m "update docs" diff --git a/docs/source/bin/index.rst b/docs/source/bin/index.rst new file mode 100644 index 00000000..6385e8ef --- /dev/null +++ b/docs/source/bin/index.rst @@ -0,0 +1,37 @@ +Client Applications +=================== + +Data Manipulation +----------------- + +.. toctree:: + :titlesonly: + + memexist — Check for the existence of a key + memcat — "cat" data from a server + memcp — "cp" files to a server + memtouch — "touch" a key + memrm – "rm" a key + +Tests and Analysis +------------------ + +.. toctree:: + :titlesonly: + + memaslap + memslap + memping – Ping a server + memstat – Gather statistics from a server + memerror — Translate libmemcached error codes + memparse — Parse and validate an option string + memcapable — Check a server's capabilities and compatibility + +Server Administration +--------------------- + +.. toctree:: + :titlesonly: + + memdump — Dump a server's data + memflush — Flush a server (erase all cached data) diff --git a/docs/source/bin/memaslap.rst b/docs/source/bin/memaslap.rst index 61060675..240c9281 100644 --- a/docs/source/bin/memaslap.rst +++ b/docs/source/bin/memaslap.rst @@ -2,8 +2,6 @@ memaslap - Load testing and benchmarking a server ================================================== - --------- SYNOPSIS -------- @@ -15,15 +13,13 @@ memaslap [options] .. envvar:: MEMCACHED_SERVERS ------------ DESCRIPTION ----------- - :program:`memaslap` is a load generation and benchmark tool for memcached -servers. It generates configurable workload such as threads, concurrencies, +servers. It generates configurable workload such as threads, concurrency, connections, run time, overwrite, miss rate, key size, value size, get/set -proportion, expected throughput, and so on. Furthermore, it also testss data +proportion, expected throughput, and so on. Furthermore, it also tests data verification, expire-time verification, UDP, binary protocol, facebook test, replication test, multi-get and reconnection, etc. @@ -36,91 +32,57 @@ value size distribution, and command distribution by itself. You can specify servers via the :option:`memslap --servers` option or via the environment variable :envvar:`MEMCACHED_SERVERS`. - --------- FEATURES -------- - Memslap is developed to for the following purposes: - Manages network connections with libevent asynchronously. - - Set both TCP and UDP up to use non-blocking IO. - - Improves parallelism: higher performance in multi-threads environments. - - Improves time efficiency: faster processing speed. - - Generates key and value more efficiently; key size distribution and value size distribution are configurable. - - Supports get, multi-get, and set commands; command distribution is configurable. - - Supports controllable miss rate and overwrite rate. - - Supports data and expire-time verification. - - Supports dumping statistic information periodically. - - Supports thousands of TCP connections. - - Supports binary protocol. - - Supports facebook test (set with TCP and multi-get with UDP) and replication test. - - - -------- DETAILS ------- - Effective implementation of network. ____________________________________ - For memaslap, both TCP and UDP use non-blocking network IO. All the network events are managed by libevent as memcached. The network module of memaslap is similar to memcached. Libevent can ensure memaslap can handle network very efficiently. - Effective implementation of multi-threads and concurrency _________________________________________________________ - Memslap has the similar implementation of multi-threads to memcached. Memslap creates one or more self-governed threads; -each thread is bound with one CPU core if the system testss setting CPU +each thread is bound with one CPU core if the system tests setting CPU core affinity. In addition, each thread has a libevent to manage the events of the network; -each thread has one or more self-governed concurrencies; and each -concurrency has one or more socket connections. All the concurrencies don’t +each thread has one or more self-governed concurrency; and each +concurrency has one or more socket connections. All the concurrent tasks don't communicate with each other even though they are in the same thread. Memslap can create thousands of socket connections, and each @@ -131,12 +93,10 @@ socket connection at any given time. Users can specify the number of concurrency and socket connections of each concurrency according to their expected workload. - Effective implementation of generating key and value ____________________________________________________ - -In order to improve time efficiency and space efficiency, +In order to improve time efficiency and space efficiency, memaslap creates a random characters table with 10M characters. All the suffixes of keys and values are generated from this random characters table. @@ -145,10 +105,10 @@ of the string to identify a string. It can save much memory. Each key contains two parts, a prefix and a suffix. The prefix is an uint64_t, 8 bytes. In order to verify the data set before, memaslap need to ensure each key is unique, so it uses the prefix to identify -a key. The prefix cannot include illegal characters, such as ‘\r’, ‘\n’, -‘\0’ and ‘ ‘. And memaslap has an algorithm to ensure that. +a key. The prefix cannot include illegal characters, such as '\r', '\n', +'\0' and ' '. And memaslap has an algorithm to ensure that. -Memslap doesn’t generate all the objects (key-value pairs) at +Memslap doesn't generate all the objects (key-value pairs) at the beginning. It only generates enough objects to fill the task window (default 10K objects) of each concurrency. Each object has the following basic information, key prefix, key suffix offset in the character table, key @@ -159,12 +119,10 @@ object from the window to do set operation or get operation. At the same time, each concurrency kicks objects out of its window and adds new object into it. - Simple but useful task scheduling _________________________________ - -Memslap uses libevent to schedule all the concurrencies of +Memslap uses libevent to schedule all concurrent tasks of threads, and each concurrency schedules tasks based on the local task window. Memslap assumes that if each concurrency keeps the same key distribution, value distribution and commands distribution, from @@ -191,11 +149,9 @@ cache layer. It causes the first cache layer to miss. So the user can specify the window size to get the expected miss rate of the first cache layer. - Useful implementation of multi-servers , UDP, TCP, multi-get and binary protocol ________________________________________________________________________________ - Because each thread is self-governed, memaslap can assign different threads to handle different memcached servers. This is just one of the ways in which memaslap tests multiple servers. The only @@ -209,7 +165,7 @@ By default, Memslap does single get. If the user specifies multi-get option, memaslap will collect enough get commands and pack and send the commands together. -Memslap testss both the ASCII protocol and binary protocol, +Memslap tests both the ASCII protocol and binary protocol, but it runs on the ASCII protocol by default. Memslap by default runs on the TCP protocol, but it also tests UDP. Because UDP is unreliable, dropped packages and out-of-order @@ -219,48 +175,29 @@ one command from the server and reorders the response data. If some packages get lost, the waiting timeout mechanism can ensure half-baked packages will be discarded and the next command will be sent. - - ------ USAGE ----- - Below are some usage samples: - memaslap -s 127.0.0.1:11211 -S 5s - - memaslap -s 127.0.0.1:11211 -t 2m -v 0.2 -e 0.05 -b - - memaslap -s 127.0.0.1:11211 -F config -t 2m -w 40k -S 20s -o 0.2 - - memaslap -s 127.0.0.1:11211 -F config -t 2m -T 4 -c 128 -d 20 -P 40k - - memaslap -s 127.0.0.1:11211 -F config -t 2m -d 50 -a -n 40 - - memaslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m - - memaslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m -p 2 - - The user must specify one server at least to run memaslap. The rest of the parameters have default values, as shown below: -Thread number = 1 Concurrency = 16 +Thread number = 1 Concurrency = 16 Run time = 600 seconds Configuration file = NULL @@ -268,33 +205,30 @@ Key size = 64 Value size = 1024 Get/set = 9:1 Window size = 10k -Execute number = 0 Single get = true +Execute number = 0 Single get = true -Multi-get = false Number of sockets of each concurrency = 1 +Multi-get = false Number of sockets of each concurrency = 1 Reconnect = false Data verification = false -Expire-time verification = false ASCII protocol = true - -Binary protocol = false Dumping statistic information +Expire-time verification = false ASCII protocol = true -periodically = false +Binary protocol = false Dumping statistic information periodically = false Overwrite proportion = 0% UDP = false -TCP = true Limit throughput = false +TCP = true Limit throughput = false -Facebook test = false Replication test = false +Facebook test = false Replication test = false Key size, value size and command distribution. ______________________________________________ - All the distributions are read from the configuration file specified by user -with “—cfg_cmd” option. If the user does not specify a configuration file, +with "—cfg_cmd" option. If the user does not specify a configuration file, memaslap will run with the default distribution (key size = 64, value size = 1024, get/set = 9:1). For information on how to edit the -configuration file, refer to the “Configuration File” section. +configuration file, refer to the "Configuration File" section. The minimum key size is 16 bytes; the maximum key size is 250 bytes. The precision of proportion is 0.001. The proportion of distribution will be @@ -303,19 +237,17 @@ rounded to 3 decimal places. The minimum value size is 1 bytes; the maximum value size is 1M bytes. The precision of proportion is 0.001. The proportion of distribution will be rounded to 3 decimal places. -Currently, memaslap only testss set and get commands. And it +Currently, memaslap only tests set and get commands. And it testss 100% set and 100% get. For 100% get, it will preset some objects to the server. - Multi-thread and concurrency ____________________________ - The high performance of memaslap benefits from the special -schedule of thread and concurrency. It’s important to specify the proper +schedule of thread and concurrency. It's important to specify the proper number of them. The default number of threads is 1; the default number of -concurrency is 16. The user can use “—threads” and “--concurrency” to +concurrency is 16. The user can use "—threads" and "--concurrency" to specify these variables. If the system tests setting CPU affinity and the number of threads @@ -359,19 +291,17 @@ limit the performance of memaslap, there are two ways to do this: Decrease the number of threads and concurrencies. -Use the option “--tps” that memaslap +Use the option "--tps" that memaslap provides to limit the throughput. This option allows the user to get the expected throughput. For example, assume that the maximum throughput is 50 kops/s for a specific configuration, you can specify the throughput equal to or less than the maximum -throughput using “--tps” option. - +throughput using "--tps" option. Window size ___________ - Most of the time, the user does not need to specify the window size. The default window size is 10k. For Schooner Memcached, the user can specify different window sizes to get different cache miss rates based on the test @@ -416,11 +346,9 @@ The formula for calculating window size for cache miss rate 5%: cache_size / concurrency / (key_size + value_size) \* 0.7 - Verification ____________ - Memslap testss both data verification and expire-time verification. The user can use "--verify=" or "-v" to specify the proportion of data verification. In theory, it testss 100% data verification. The @@ -429,16 +357,14 @@ expire-time verification. In theory, it testss 100% expire-time verification. Specify the "--verbose" options to get more detailed error information. -For example: --exp_verify=0.01 –verify=0.1 , it means that 1% of the objects +For example: --exp_verify=0.01 –verify=0.1 , it means that 1% of the objects set with expire-time, 10% of the objects gotten will be verified. If the objects are gotten, memaslap will verify the expire-time and value. - multi-servers and multi-config _______________________________ - Memslap testss multi-servers based on self-governed thread. There is a limitation that the number of servers cannot be greater than the number of threads. Memslap assigns one thread to handle one @@ -456,17 +382,15 @@ and 4 handle server 1 (10.1.1.2); and thread 2 and 5 handle server 2 All the threads and concurrencies in memaslap are self-governed. -So is memaslap. The user can start up several +So is memaslap. The user can start up several memaslap instances. The user can run memaslap on different client machines to communicate with the same memcached server at the same. It is recommended that the user start different memaslap on different machines using the same configuration. - Run with execute number mode or time mode _________________________________________ - The default memaslap runs with time mode. The default run time is 10 minutes. If it times out, memaslap will exit. Do not specify both execute number mode and time mode at the same time; just @@ -478,11 +402,9 @@ For example: --execute_number=100000 (It means that after running 100000 commands, the test will exit.) - Dump statistic information periodically. ________________________________________ - The user can use "--stat_freq=" or "-S" to specify the frequency. For example: @@ -492,13 +414,11 @@ For example: Memslap will dump the statistics of the commands (get and set) at the frequency of every 20 seconds. -For more information on the format of dumping statistic information, refer to “Format of Output” section. - +For more information on the format of dumping statistic information, refer to "Format of Output" section. Multi-get _________ - The user can use "--division=" or "-d" to specify multi-get keys count. Memslap by default does single get with TCP. Memslap also testss data verification and expire-time verification for multi-get. @@ -506,36 +426,32 @@ verification and expire-time verification for multi-get. Memslap testss multi-get with both TCP and UDP. Because of the different implementation of the ASCII protocol and binary protocol, there are some differences between the two. For the ASCII protocol, -memaslap sends one “multi-get” to the server once. For the +memaslap sends one "multi-get" to the server once. For the binary protocol, memaslap sends several single get commands -together as “multi-get” to the server. - +together as "multi-get" to the server. UDP and TCP ___________ - Memslap testss both UDP and TCP. For TCP, memaslap does not reconnect the memcached server if socket connections are lost. If all the socket connections are lost or memcached server crashes, -memaslap will exit. If the user specifies the “--reconnect” +memaslap will exit. If the user specifies the "--reconnect" option when socket connections are lost, it will reconnect them. -User can use “--udp” to enable the UDP feature, but UDP comes with some +User can use "--udp" to enable the UDP feature, but UDP comes with some limitations: UDP cannot set data more than 1400 bytes. -UDP is not testsed by the binary protocol because the binary protocol of +UDP is not tested by the binary protocol because the binary protocol of memcached does not tests that. -UDP doesn’t tests reconnection. - +UDP doesn't tests reconnection. Facebook test _____________ - Set data with TCP and multi-get with UDP. Specify the following options: "--facebook --division=50" @@ -555,13 +471,11 @@ objects once with the UDP socket. If you specify "--division=50", the key size must be less that 25 bytes because the UDP packet size is 1400 bytes. - Replication test ________________ - For replication test, the user must specify at least two memcached servers. -The user can use “—rep_write=” option to enable feature. +The user can use "—rep_write=" option to enable feature. For example: @@ -575,11 +489,9 @@ memaslap will only get objects from server 1. If server 0 comes back to life again, memaslap will reconnect server 0. If both server 0 and server 1 crash, memaslap will exit. - Supports thousands of TCP connections _____________________________________ - Start memaslap with "--conn_sock=" or "-n" to enable this feature. Make sure that your system can tests opening thousands of files and creating thousands of sockets. However, this feature does not tests @@ -594,11 +506,9 @@ thread has 16 concurrencies, each concurrency has 128 TCP socket connections, and the total number of TCP socket connections is 128 \* 128 = 16384. - Supports binary protocol ________________________ - Start memaslap with "--binary" or "-B" options to enable this feature. It testss all the above features except UDP, because the latest memcached 1.3.3 does not implement binary UDP protocol. @@ -610,22 +520,17 @@ For example: Since memcached 1.3.3 doesn't implement binary UDP protocol, memaslap does not tests UDP. In addition, memcached 1.3.3 does not tests multi-get. If you specify "--division=50" option, it just sends 50 get -commands together as “mulit-get” to the server. - +commands together as "multi-get" to the server. - ------------------- Configuration file ------------------ - This section describes the format of the configuration file. By default when no configuration file is specified memaslap reads the default one located at ~/.memaslap.cnf. Below is a sample configuration file: - .. code-block:: perl --------------------------------------------------------------------------- @@ -700,94 +605,58 @@ Below is a sample configuration file: 0 0.1 1.0 0.9 - - ----------------- Format of output ---------------- - At the beginning, memaslap displays some configuration information as follows: - servers : 127.0.0.1:11211 - - threads count: 1 - - concurrency: 16 - - run time: 20s - - windows size: 10k - - set proportion: set_prop=0.10 - - get proportion: get_prop=0.90 - - Where _____ - - servers : "servers" The servers used by memaslap. - - threads count The number of threads memaslap runs with. - - concurrency The number of concurrencies memaslap runs with. - - run time How long to run memaslap. - - windows size The task window size of each concurrency. - - set proportion The proportion of set command. - - get proportion The proportion of get command. - - The output of dynamic statistics is something like this: - .. code-block:: perl --------------------------------------------------------------------------------------------------------------------------------- @@ -798,18 +667,16 @@ The output of dynamic statistics is something like this: 95.43 177.29 Global 20 1257935 62896 71.8 0 26 3791 224 117.79 192.60 - - Set Statistics + Type Time(s) Ops TPS(ops/s) Net(M/s) Get_miss Min(us) Max(us) Avg(us) Std_dev Geo_dist Period 5 38425 7685 7.3 0 42 628 240 88.05 220.21 Global 20 139780 6989 8.0 0 37 3790 253 117.93 224.83 - - Total Statistics + Type Time(s) Ops TPS(ops/s) Net(M/s) Get_miss Min(us) Max(us) Avg(us) Std_dev Geo_dist Period 5 384252 76850 72.5 0 27 2198 207 @@ -818,100 +685,67 @@ The output of dynamic statistics is something like this: 117.93 195.60 --------------------------------------------------------------------------------------------------------------------------------- - - Where _____ - - Get Statistics Statistics information of get command - - Set Statistics Statistics information of set command - - Total Statistics Statistics information of both get and set command - - Period Result within a period - - Global Accumulated results - - Ops Total operations - - TPS Throughput, operations/second - - Net The rate of network - - Get_miss - How many objects can’t be gotten - - + How many objects can't be gotten Min The minimum response time - - Max The maximum response time - - Avg: The average response time - - Std_dev Standard deviation of response time - - Geo_dist Geometric distribution based on natural exponential function - - At the end, memaslap will output something like this: - .. code-block:: perl --------------------------------------------------------------------------------------------------------------------------------- @@ -965,170 +799,112 @@ At the end, memaslap will output something like this: Run time: 20.0s Ops: 1397754 TPS: 69817 Net_rate: 59.4M/s --------------------------------------------------------------------------------------------------------------------------------- - - Where _____ - - Get Statistics Get statistics of response time - - Set Statistics Set statistics of response time - - Total Statistics Both get and set statistics of response time - - Min The accumulated and minimum response time - - Max The accumulated and maximum response time - - Avg The accumulated and average response time - - Std Standard deviation of response time - - Log2 Dist Geometric distribution based on logarithm 2 - - cmd_get Total get commands done - - cmd_set Total set commands done - - get_misses - How many objects can’t be gotten from server - - + How many objects can't be gotten from server verify_misses - How many objects need to verify but can’t get them - - + How many objects need to verify but can't get them verify_failed How many objects with insistent value - - expired_get How many objects are expired but we get them - - unexpired_unget - How many objects are unexpired but we can’t get them - - + How many objects are unexpired but we can't get them written_bytes Total written bytes - - read_bytes Total read bytes - - object_bytes Total object bytes - - packet_disorder How many UDP packages are disorder - - packet_drop How many UDP packages are lost - - udp_timeout How many times UDP time out happen - - Run time Total run time - - Ops Total operations - - TPS Throughput, operations/second - - Net_rate The average rate of network - - - - -------- OPTIONS ------- - -s, --servers= List one or more servers to connect. Servers count must be less than threads count. e.g.: --servers=localhost:1234,localhost:11211 @@ -1178,17 +954,17 @@ OPTIONS The proportion of objects need overwrite, e.g.: --overwrite=0.01. Default never overwrite object. --R, --reconnect +-R, --reconnect Reconnect tests, when connection is closed it will be reconnected. --U, --udp +-U, --udp UDP tests, default memaslap uses TCP, TCP port and UDP port of server must be same. --a, --facebook +-a, --facebook Whether it enables facebook test feature, set with TCP and multi-get with UDP. --B, --binary +-B, --binary Whether it enables binary protocol. Default with ASCII protocol. -P, --tps= @@ -1197,21 +973,18 @@ OPTIONS -p, --rep_write= The first nth servers can write data, e.g.: --rep_write=2. --b, --verbose +-b, --verbose Whether it outputs detailed information when verification fails. --h, --help +-h, --help Display this message and then exit. --V, --version +-V, --version Display the version of the application and then exit. - --------- EXAMPLES -------- - memaslap -s 127.0.0.1:11211 -S 5s memaslap -s 127.0.0.1:11211 -t 2m -v 0.2 -e 0.05 -b @@ -1226,27 +999,9 @@ memaslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m memaslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m -p 2 - ----- -HOME ----- - - -To find out more information please check: -`http://libmemcached.org/ `_ - - -------- -AUTHORS -------- - - -Mingqiang Zhuang (Schooner Technolgy) -Brian Aker, - - --------- SEE ALSO -------- -:manpage:`memcached(1)` :manpage:`libmemcached(3)` +.. only:: man + + :manpage:`memcached(1)` :manpage:`libmemcached(3)` diff --git a/docs/source/bin/memcapable.rst b/docs/source/bin/memcapable.rst index 0e4f33e3..d1a4f7a4 100644 --- a/docs/source/bin/memcapable.rst +++ b/docs/source/bin/memcapable.rst @@ -1,78 +1,52 @@ -======================================================================= -memcapable - Checking a Memcached server capibilities and compatibility -======================================================================= +memcapable +========== --------- SYNOPSIS -------- +.. program:: memcapable + memcapable [options] -.. program:: memcapable +Check a memcached server's capabilities and compatibility. ------------ DESCRIPTION ----------- :program:`memcapable` connects to the specified memcached server and tries to -determine its capabilities by running the various commands and verifying -the response. - +determine its capabilities by running various commands and verifying the response. ------------ -LIMITATIONS ------------ - - -The current version of memcapable will only verify the binary protocol. - - -------- OPTIONS ------- - .. option:: -h hostname - -Specify the hostname to connect to. The default is \ *localhost*\ -.. option:: -p port - -Specify the port number to connect to. The default is \ *11211*\ - -.. option:: -c - -Generate a coredump when it detects an error from the server. - -.. option:: -v - -Print out the comparison when it detects an error from the server. - -.. option:: -t n - -Set the timeout from an IO operation to/from the server to \ *n*\ seconds. - + Specify the hostname to connect to. The default is \ *localhost*\ . ----- -HOME ----- +.. option:: -p port + Specify the port number to connect to. The default is \ *11211*\ . -To find out more information please check: -`http://libmemcached.org/ `_ +.. option:: -c + :manpage:`abort(3)` when detecting an error from the server. ------- -AUTHOR ------- +.. option:: -v + Print out the comparison when it detects an error from the server. -Trond Norbye, +.. option:: -t n + Set the timeout for an IO operation to/from the server to \ *n*\ seconds. --------- SEE ALSO -------- -:manpage:`memcached(1)` :manpage:`libmemcached(3)` +.. only:: man + + :manpage:`memcached(1)` + :manpage:`libmemcached(3)` + +.. only:: html + * :doc:`/libmemcached` diff --git a/docs/source/bin/memcat.rst b/docs/source/bin/memcat.rst index b8612a92..aa6cc540 100644 --- a/docs/source/bin/memcat.rst +++ b/docs/source/bin/memcat.rst @@ -1,73 +1,53 @@ -================================= -memcat - "cat" data from a server -================================= +memcat +====== - - --------- SYNOPSIS -------- -memcat [options] key - -Copy a set of keys to stdout - - .. program:: memcat +memcat [options] key [key...] + +Read and output the value of one key or the values of a set of keys. ------------ DESCRIPTION ----------- +:program:`memcat` reads and outputs the value of a single or a set of keys +stored in a :manpage:`memcached(1)` server. -:program:`memcat` outputs to stdout the value a single or multiple set of keys -stored in a memcached(1) server. If any key is not found an error is returned. - -It is similar to the standard UNIX cat(1) utility. +If any key is not found an error is returned. +It is similar to the standard UNIX :manpage:`cat(1)` utility. -------- OPTIONS ------- +.. include:: options/all.rst +.. include:: options/common.rst +.. include:: options/hash.rst -You can specify servers via the option: - -.. cmdoption:: --servers - -or via the environment variable: +.. option:: --flag -.. envvar:: `MEMCACHED_SERVERS` + Display stored flags. -For a full list of operations run the tool with the option: - -.. cmdoption:: --help - - ----- -HOME ----- - - -To find out more information please check: -`http://libmemcached.org/ `_ - - ------- -AUTHOR ------- - - -Brian Aker, +ENVIRONMENT +----------- -Mark Atwood +.. envvar:: MEMCACHED_SERVERS + Specify a list of servers. --------- SEE ALSO -------- +.. only:: man + + :manpage:`memcached(1)` + :manpage:`libmemcached(3)` + :manpage:`libmemcached_configuration(3)` -:manpage:`memcached(1)` :manpage:`libmemcached(3)` +.. only:: html + * :doc:`/libmemcached` + * :doc:`/libmemcached/configuration` diff --git a/docs/source/bin/memcp.rst b/docs/source/bin/memcp.rst index a85eed8b..deea12a7 100644 --- a/docs/source/bin/memcp.rst +++ b/docs/source/bin/memcp.rst @@ -1,75 +1,69 @@ -============================= -memcp - Copy data to a server -============================= +memcp +===== - -Copies files to a collection of memcached servers - - --------- SYNOPSIS -------- -memcp [options] [file] [server] - .. program:: memcp +memcp [options] \-\-servers + +Copy files to a collection of memcached servers. ------------ DESCRIPTION ----------- +:program:`memcp` copies one or more files into :manpage:`memcached(1)` servers. +It is similar to the standard UNIX :manpage:`cp(1)` command. -:program:`memcp` copies one or more files into memcached(1) servers. -It is similar to the standard UNIX cp(1) command. - -The key names will be the names of the files, -without any directory path. +The key names will be the names of the files, without any directory path. - -------- OPTIONS ------- +.. include:: options/all.rst +.. include:: options/common.rst +.. include:: options/expire.rst +.. include:: options/flag.rst +.. include:: options/hash.rst +.. include:: options/sasl.rst +.. include:: options/udp.rst -You can specify servers via the option: +.. option:: --buffer -.. option:: --servers + Enable internal buffering of commands. -or via the environment variable: +.. option:: --set -.. envvar:: `MEMCACHED_SERVERS` + Issue *SET* command(s). This is the default mode. + See also :option:`--add` and :option:`--replace`. -If you do not specify either these, the final value in the command line list is the name of a server(s). +.. option:: --add -For a full list of operations run the tool with the option: + Issue *ADD* command(s). -.. option:: --help +.. option:: --replace + Issue *REPLACE* command(s). ----- -HOME ----- - - -To find out more information please check: -`http://libmemcached.org/ `_ - - ------- -AUTHOR ------- - - -Brian Aker, +ENVIRONMENT +----------- -Mark Atwood, +.. envvar:: MEMCACHED_SERVERS + Specify the list of servers. --------- SEE ALSO -------- +.. only:: man + + :manpage:`memcached(1)` + :manpage:`libmemcached(3)` + :manpage:`memcached_behavior(3)` -:manpage:`memcached(1)` :manpage:`libmemcached(3)` +.. only:: html + * :doc:`/libmemcached` + * :doc:`/libmemcached/configuration` + * :doc:`/libmemcached/memcached_behavior` diff --git a/docs/source/bin/memdump.rst b/docs/source/bin/memdump.rst index 6a8e12da..07eb2fff 100644 --- a/docs/source/bin/memdump.rst +++ b/docs/source/bin/memdump.rst @@ -1,58 +1,46 @@ -============================= -memdump - Dumping your server -============================= +memdump +======= - -Dump a list of keys from a server. - - --------- SYNOPSIS -------- +.. program:: memdump + memdump [options] -.. program:: memdump +Dump a list of keys from a server. ------------ DESCRIPTION ----------- - -:program:`memdump` dumps a list of "keys" from all servers that -it is told to fetch from. Because memcached does not guarentee to +:program:`memdump` dumps a list of "keys" from all servers that +it is told to fetch from. Because memcached does not guarantee to provide all keys it is not possible to get a complete "dump". - -------- OPTIONS ------- +.. include:: options/all.rst +.. include:: options/common.rst +.. include:: options/hash.rst +.. include:: options/sasl.rst -For a full list of operations run the tool with option: - -.. option:: --help - - ----- -HOME ----- - - -To find out more information please check: -`http://libmemcached.org/ `_ +ENVIRONMENT +----------- +.. envvar:: MEMCACHED_SERVERS ------- -AUTHOR ------- + Specify the list of servers. +SEE ALSO +-------- -Brian Aker, +.. only:: man + :manpage:`memcached(1)` + :manpage:`libmemcached(3)` --------- -SEE ALSO --------- +.. only:: html -:manpage:`memcached(1)` :manpage:`libmemcached(3)` + * :doc:`/libmemcached` + * :doc:`/libmemcached/memcached_dump` diff --git a/docs/source/bin/memerror.rst b/docs/source/bin/memerror.rst index 53fc09c4..525aaf63 100644 --- a/docs/source/bin/memerror.rst +++ b/docs/source/bin/memerror.rst @@ -1,58 +1,42 @@ -============================================== -memerror - translate an error code to a string -============================================== +memerror +======== - -Translates a memcached error code into a string - - --------- SYNOPSIS -------- -memerror [options] [error code] - .. program:: memerror ------------ +memerror [options] + +Translate a memcached error code into a string. + DESCRIPTION ----------- - -:program:`memerror` translates an error code from libmemcached(3) into a human +:program:`memerror` translates an error code from `libmemcached` into a human readable string. - -------- OPTIONS ------- +.. include:: options/all.rst -For a full list of operations run the tool with option: - -.. option:: --help - - ----- -HOME ----- - - -To find out more infoerroration please check: -`http://libmemcached.org/ `_ - - ------- -AUTHOR ------- - +ENVIRONMENT +----------- -Brian Aker, +.. envvar:: MEMCACHED_SERVERS + Specify the list of servers. --------- SEE ALSO -------- +.. only:: man + + :manpage:`memcached(1)` + :manpage:`libmemcached(3)` + +.. only:: html -:manpage:`memcached(1)` :manpage:`libmemcached(3)` + * :doc:`/libmemcached` + * :doc:`/libmemcached/index_errors` diff --git a/docs/source/bin/memexist.rst b/docs/source/bin/memexist.rst index 22c3a4d8..1859c838 100644 --- a/docs/source/bin/memexist.rst +++ b/docs/source/bin/memexist.rst @@ -1,64 +1,44 @@ -=========================================== -memexist - Check for the existance of a key -=========================================== +memexist +======== - --------- SYNOPSIS -------- -memexist [options] [key] - .. program:: memexist +memexist [options] + +Check for the existence of a key. ------------ DESCRIPTION ----------- -:program:`memexist` checks for the existance of a key within a cluster. - +:program:`memexist` checks for the existence of a key within a cluster. -------- OPTIONS ------- +.. include:: options/all.rst +.. include:: options/common.rst +.. include:: options/hash.rst +.. include:: options/sasl.rst -You can specify servers via the option: - -.. option:: --servers - -or via the environment variable: - -.. envvar:: `MEMCACHED_SERVERS` - -If you do not specify either these, the final value in the command line list is the name of a server(s). - -For a full list of operations run the tool with the option: - -.. option:: --help - - ----- -HOME ----- - - -To find out more information please check: -`http://libmemcached.org/ `_ - - ------- -AUTHOR ------- +ENVIRONMENT +----------- +.. envvar:: MEMCACHED_SERVERS -Brian Aker, + Specify the list of servers. --------- SEE ALSO -------- +.. only:: man + + :manpage:`memcached(1)` + :manpage:`libmemcached(3)` -:manpage:`memcached(1)` :manpage:`libmemcached(3)` +.. only:: html + * :doc:`/libmemcached` + * :doc:`/libmemcached/memcached_exist` diff --git a/docs/source/bin/memflush.rst b/docs/source/bin/memflush.rst index a2b9894d..ed84d9a9 100644 --- a/docs/source/bin/memflush.rst +++ b/docs/source/bin/memflush.rst @@ -1,12 +1,6 @@ -======================================= -memflush - flush all data from a server -======================================= +memflush +======== - -Reset a server or list of servers - - --------- SYNOPSIS -------- @@ -14,54 +8,67 @@ memflush [options] .. program:: memflush ------------ +Reset a server or list of servers + DESCRIPTION ----------- +:program:`memflush` resets the contents of :manpage:`memcached(1)` servers. -:program:`memflush` resets the contents of memcached(1) servers. -This means that all data in the specified servers will be deleted. +.. warning:: + This means that all data in the specified servers will be deleted. -------- OPTIONS ------- +.. include:: options/all.rst +.. include:: options/common.rst +.. include:: options/sasl.rst +.. include:: options/expire.rst -You can specify servers via the option: +.. note:: -.. option:: --servers + Using an expiration time (period), all keys, which have not bean updated until expiration will cease to exist. -or via the environment variable: + Quoting the `memcached protocol documentation`_, it states: -.. envvar:: `MEMCACHED_SERVERS` + Its effect is to invalidate all + existing items immediately (by default) or after the expiration + specified. After invalidation none of the items will be returned in + response to a retrieval command (unless it's stored again under the + same key *after* flush_all has invalidated the items). -For a full list of operations run the tool with option: + The most precise + definition of what flush_all does is the following: it causes all + items whose update time is earlier than the time at which flush_all + was set to be executed to be ignored for retrieval purposes. -.. option:: --help + The intent of flush_all with a delay, was that in a setting where you + have a pool of memcached servers, and you need to flush all content, + you have the option of not resetting all memcached servers at the + same time (which could e.g. cause a spike in database load with all + clients suddenly needing to recreate content that would otherwise + have been found in the memcached daemon). +.. _memcached protocol documentation: https://github.com/memcached/memcached/blob/master/doc/protocol.txt ----- -HOME ----- - - -To find out more information please check: -`http://libmemcached.org/ `_ - +ENVIRONMENT +----------- ------- -AUTHOR ------- +.. envvar:: MEMCACHED_SERVERS + Specify the list of servers. -Brian Aker, +SEE ALSO +-------- -Mark Atwood +.. only:: man + :manpage:`memcached(1)` + :manpage:`libmemcached(3)` --------- -SEE ALSO --------- +.. only:: html -:manpage:`memcached(1)` :manpage:`libmemcached(3)` +* :doc:`/libmemcached` +* :doc:`/libmemcached/memcached_flush` diff --git a/docs/source/bin/memparse.rst b/docs/source/bin/memparse.rst index 5aca71fb..4a6f4135 100644 --- a/docs/source/bin/memparse.rst +++ b/docs/source/bin/memparse.rst @@ -1,56 +1,36 @@ -================================= -memparse - Parse an option string -================================= +memparse +======== - -Used to validate an option string - - --------- SYNOPSIS -------- -memparse [options] "option string" - .. program:: memparse +memparse