From: Brian Aker Date: Sat, 19 Mar 2011 00:08:39 +0000 (-0700) Subject: Merge in additions to the scanner. X-Git-Tag: 0.51~15^2~75^2~19 X-Git-Url: https://git.m6w6.name/?a=commitdiff_plain;h=b95041363ea69ccfb9eacb6bba1852814c32b4ee;p=m6w6%2Flibmemcached Merge in additions to the scanner. --- diff --git a/libmemcached/options/parser.yy b/libmemcached/options/parser.yy index 8ac4d14d..d4275a09 100644 --- a/libmemcached/options/parser.yy +++ b/libmemcached/options/parser.yy @@ -1,38 +1,21 @@ /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: - * - * Libmemcached library * - * Copyright (C) 2011 Data Differential, http://datadifferential.com/ - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * * The names of its contributors may not be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Libmemcached Scanner and Parser * + * Copyright (C) 2011 DataDifferental, http://datadifferential.com + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . */ %{ @@ -49,7 +32,9 @@ inline int libmemcached_error(YYLTYPE *locp, type_st *parser, yyscan_t *scanner, const char *str) { +#if 0 std::cerr << str << std::endl; +#endif return 0; } @@ -74,9 +59,61 @@ inline int libmemcached_error(YYLTYPE *locp, type_st *parser, yyscan_t *scanner, %token SERVER %token SERVERS -%token TCPNODELAY %token UNKNOWN + +%token DASH_OPTION + +/* All behavior options */ +%token AUTO_EJECT_HOSTS +%token BINARY_PROTOCOL +%token BUFFER_REQUESTS +%token CACHE_LOOKUPS +%token CONNECT_TIMEOUT +%token _CORK +%token DISTRIBUTION +%token HASH +%token HASH_WITH_PREFIX_KEY +%token IO_BYTES_WATERMARK +%token IO_KEY_PREFETCH +%token IO_MSG_WATERMARK +%token KETAMA +%token KETAMA_HASH +%token KETAMA_WEIGHTED +%token NOREPLY +%token NUMBER_OF_REPLICAS +%token POLL_TIMEOUT +%token RANDOMIZE_REPLICA_READ +%token RCV_TIMEOUT +%token RETRY_TIMEOUT +%token SERVER_FAILURE_LIMIT +%token SND_TIMEOUT +%token SOCKET_RECV_SIZE +%token SOCKET_SEND_SIZE +%token SORT_HOSTS +%token SUPPORT_CAS +%token _TCP_NODELAY +%token _TCP_KEEPALIVE +%token _TCP_KEEPIDLE +%token USER_DATA +%token USE_UDP %token VERIFY_KEY + +/* Hash types */ +%token MD5 +%token CRC +%token FNV1_64 +%token FNV1A_64 +%token FNV1_32 +%token FNV1A_32 +%token HSIEH +%token MURMUR +%token JENKINS + +/* Distributions */ +%token CONSISTENT +%token MODULA +%token RANDOM + %nonassoc ',' %nonassoc '=' @@ -88,13 +125,15 @@ inline int libmemcached_error(YYLTYPE *locp, type_st *parser, yyscan_t *scanner, %token IPADDRESS_WITH_PORT %type server +%type distribution +%type hash %% statement: - expression + DASH_OPTION expression { } - | statement expression + | statement ' ' DASH_OPTION expression { } ; @@ -105,19 +144,146 @@ expression: (void) memcached_server_add_parsed(parser->memc, $3.c_str, $3.length, $3.port, 0); } | SERVERS '=' server_list - { } - | TCPNODELAY + { + } + | behaviors + ; + +behaviors: + AUTO_EJECT_HOSTS + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_AUTO_EJECT_HOSTS, 1); + } + | BINARY_PROTOCOL + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1); + } + | BUFFER_REQUESTS + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS, 1); + } + | CACHE_LOOKUPS + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_CACHE_LOOKUPS, 1); + } + | CONNECT_TIMEOUT '=' NUMBER + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_CONNECT_TIMEOUT, $3); + } + | _CORK + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_CORK, 1); + } + | DISTRIBUTION '=' distribution + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_DISTRIBUTION, $3); + } + | HASH '=' hash + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_DISTRIBUTION, $3); + } + | HASH_WITH_PREFIX_KEY + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_HASH_WITH_PREFIX_KEY, 1); + } + | IO_BYTES_WATERMARK '=' NUMBER + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_IO_MSG_WATERMARK, $3); + } + | IO_KEY_PREFETCH '=' NUMBER + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_IO_KEY_PREFETCH, $3); + } + | IO_MSG_WATERMARK '=' NUMBER + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_IO_MSG_WATERMARK, $3); + } + | KETAMA + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_KETAMA, true); + } + | KETAMA_HASH + { + } + | KETAMA_WEIGHTED + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_KETAMA_WEIGHTED, true); + } + | NOREPLY + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_NOREPLY, 1); + } + | NUMBER_OF_REPLICAS '=' NUMBER + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_NUMBER_OF_REPLICAS, $3); + } + | POLL_TIMEOUT '=' NUMBER + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_POLL_TIMEOUT, $3); + } + | RANDOMIZE_REPLICA_READ + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_RANDOMIZE_REPLICA_READ, true); + } + | RCV_TIMEOUT '=' NUMBER + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_RCV_TIMEOUT, $3); + } + | RETRY_TIMEOUT '=' NUMBER + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_RETRY_TIMEOUT, $3); + } + | SERVER_FAILURE_LIMIT '=' NUMBER + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_SERVER_FAILURE_LIMIT, $3); + } + | SND_TIMEOUT '=' NUMBER + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_SND_TIMEOUT, $3); + } + | SOCKET_RECV_SIZE '=' NUMBER + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_SOCKET_RECV_SIZE, $3); + } + | SOCKET_SEND_SIZE '=' NUMBER + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_SOCKET_SEND_SIZE, $3); + } + | SORT_HOSTS + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_SORT_HOSTS, true); + } + | SUPPORT_CAS + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_SUPPORT_CAS, true); + } + | _TCP_NODELAY { memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_TCP_NODELAY, true); } - | VERIFY_KEY + | _TCP_KEEPALIVE + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_TCP_KEEPALIVE, true); + } + | _TCP_KEEPIDLE + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_TCP_KEEPIDLE, true); + } + | USER_DATA + { + } + | USE_UDP + { + memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_USE_UDP, true); + } + | VERIFY_KEY { memcached_behavior_set(parser->memc, MEMCACHED_BEHAVIOR_VERIFY_KEY, true); } ; server_list: - server + server { (void) memcached_server_add_parsed(parser->memc, $1.c_str, $1.length, $1.port, 0); } @@ -153,3 +319,57 @@ server: $$.port= MEMCACHED_DEFAULT_PORT; } ; + +hash: + MD5 + { + $$= MEMCACHED_HASH_MD5; + } + | CRC + { + $$= MEMCACHED_HASH_CRC; + } + | FNV1_64 + { + $$= MEMCACHED_HASH_FNV1_64; + } + | FNV1A_64 + { + $$= MEMCACHED_HASH_FNV1A_64; + } + | FNV1_32 + { + $$= MEMCACHED_HASH_FNV1_32; + } + | FNV1A_32 + { + $$= MEMCACHED_HASH_FNV1A_32; + } + | HSIEH + { + $$= MEMCACHED_HASH_HSIEH; + } + | MURMUR + { + $$= MEMCACHED_HASH_MURMUR; + } + | JENKINS + { + $$= MEMCACHED_HASH_JENKINS; + } + ; + +distribution: + CONSISTENT + { + $$= MEMCACHED_DISTRIBUTION_CONSISTENT; + } + | MODULA + { + $$= MEMCACHED_DISTRIBUTION_MODULA; + } + | RANDOM + { + $$= MEMCACHED_DISTRIBUTION_RANDOM; + } + ; diff --git a/libmemcached/options/scanner.l b/libmemcached/options/scanner.l index 77f16ebe..c8c2ea82 100644 --- a/libmemcached/options/scanner.l +++ b/libmemcached/options/scanner.l @@ -1,3 +1,22 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * + * Libmemcached Scanner and Parser + * + * Copyright (C) 2011 DataDifferental, http://datadifferential.com + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ %top{ @@ -53,21 +72,94 @@ static void get_lex_chars(char* buffer, int& result, int max_size, struct type_s %% -=|, { return yytext[0];} +=|,|[ ] { return yytext[0];} [[:digit:]]+ { yylval->number = atoi(yytext); return (NUMBER); } ([[:digit:]]*.[:digit:]+) { yylval->double_number = atof(yytext); return (FLOAT); } -[ \t\r\n] ; /* skip whitespace */ - -"--SERVER" { return SERVER; } -"--SERVERS" { return SERVERS; } -"--TCP_NODELAY" { return TCPNODELAY; } -"--TCP-NODELAY" { return TCPNODELAY; } -"--VERIFY_KEY" { return VERIFY_KEY; } -"--VERIFY-KEY" { return VERIFY_KEY; } +[\t\r\n] ; /* skip whitespace */ + +"--" { return DASH_OPTION; } + +SERVER { return SERVER; } +SERVERS { return SERVERS; } + +VERIFY_KEY { return VERIFY_KEY; } +VERIFY-KEY { return VERIFY_KEY; } +AUTO_EJECT_HOSTS { return AUTO_EJECT_HOSTS; } +AUTO-EJECT_HOSTS { return AUTO_EJECT_HOSTS; } +BINARY_PROTOCOL { return BINARY_PROTOCOL; } +BINARY-PROTOCOL { return BINARY_PROTOCOL; } +BUFFER_REQUESTS { return BUFFER_REQUESTS; } +BUFFER-REQUESTS { return BUFFER_REQUESTS; } +CACHE_LOOKUPS { return CACHE_LOOKUPS; } +CACHE-LOOKUPS { return CACHE_LOOKUPS; } +CONNECT_TIMEOUT { return CONNECT_TIMEOUT; } +CONNECT-TIMEOUT { return CONNECT_TIMEOUT; } +CORK { return _CORK; } +DISTRIBUTION { return DISTRIBUTION; } +HASH { return HASH; } +HASH_WITH_PREFIX_KEY { return HASH_WITH_PREFIX_KEY; } +HASH-WITH-PREFIX_KEY { return HASH_WITH_PREFIX_KEY; } +IO_BYTES_WATERMARK { return IO_BYTES_WATERMARK; } +IO-BYTES-WATERMARK { return IO_BYTES_WATERMARK; } +IO_KEY_PREFETCH { return IO_KEY_PREFETCH; } +IO-KEY-PREFETCH { return IO_KEY_PREFETCH; } +IO_MSG_WATERMARK { return IO_MSG_WATERMARK; } +IO-MSG-WATERMARK { return IO_MSG_WATERMARK; } +KETAMA { return KETAMA; } +KETAMA_HASH { return KETAMA_HASH; } +KETAMA-HASH { return KETAMA_HASH; } +KETAMA_WEIGHTED { return KETAMA_WEIGHTED; } +KETAMA-WEIGHTED { return KETAMA_WEIGHTED; } +NOREPLY { return NOREPLY; } +NUMBER_OF_REPLICAS { return NUMBER_OF_REPLICAS; } +NUMBER-OF-REPLICAS { return NUMBER_OF_REPLICAS; } +POLL_TIMEOUT { return POLL_TIMEOUT; } +POLL-TIMEOUT { return POLL_TIMEOUT; } +RANDOMIZE_REPLICA_READ { return RANDOMIZE_REPLICA_READ; } +RANDOMIZE-REPLICA-READ { return RANDOMIZE_REPLICA_READ; } +RCV_TIMEOUT { return RCV_TIMEOUT; } +RCV-TIMEOUT { return RCV_TIMEOUT; } +RETRY_TIMEOUT { return RETRY_TIMEOUT; } +RETRY-TIMEOUT { return RETRY_TIMEOUT; } +SERVER-FAILURE-LIMIT { return SERVER_FAILURE_LIMIT; } +SND_TIMEOUT { return SND_TIMEOUT; } +SND-TIMEOUT { return SND_TIMEOUT; } +SOCKET_RECV_SIZE { return SOCKET_RECV_SIZE; } +SOCKET-RECV-SIZE { return SOCKET_RECV_SIZE; } +SOCKET_SEND_SIZE { return SOCKET_SEND_SIZE; } +SOCKET-SEND-SIZE { return SOCKET_SEND_SIZE; } +SORT_HOSTS { return SORT_HOSTS; } +SORT-HOSTS { return SORT_HOSTS; } +SUPPORT_CAS { return SUPPORT_CAS; } +SUPPORT-CAS { return SUPPORT_CAS; } +TCP_NODELAY { return _TCP_NODELAY; } +TCP-NODELAY { return _TCP_NODELAY; } +TCP_KEEPALIVE { return _TCP_KEEPALIVE; } +TCP-KEEPALIVE { return _TCP_KEEPALIVE; } +TCP_KEEPIDLE { return _TCP_KEEPIDLE; } +TCP-KEEPIDLE { return _TCP_KEEPIDLE; } +USER_DATA { return USER_DATA; } +USER-DATA { return USER_DATA; } +USE_UDP { return USE_UDP; } +USE-UDP { return USE_UDP; } + +CONSISTENT { return CONSISTENT; } +MODULA { return MODULA; } +RANDOM { return RANDOM; } + +MD5 { return MD5; } +CRC { return CRC; } +FNV1_64 { return FNV1_64; } +FNV1A_64 { return FNV1A_64; } +FNV1_32 { return FNV1_32; } +FNV1A_32 { return FNV1A_32; } +HSIEH { return HSIEH; } +MURMUR { return MURMUR; } +JENKINS { return JENKINS; } [[:alnum:]][[:alnum:].]*[[:alpha:]]: { yylval->string.c_str = yytext; @@ -92,8 +184,7 @@ static void get_lex_chars(char* buffer, int& result, int max_size, struct type_s return IPADDRESS; } -. { - std::cerr << "Near " << yytext << std::endl; +. { return UNKNOWN; } diff --git a/libmemcached/options/symbol.h b/libmemcached/options/symbol.h index 508dacbd..161474dc 100644 --- a/libmemcached/options/symbol.h +++ b/libmemcached/options/symbol.h @@ -37,6 +37,7 @@ #pragma once +#include #include #include @@ -46,6 +47,8 @@ union YYSTYPE string_t string; server_t server; double double_number; + memcached_server_distribution_t distribution; + memcached_hash_t hash; }; typedef union YYSTYPE YYSTYPE; diff --git a/libtest/test.h b/libtest/test.h index 88cc0115..b7a2f5ee 100644 --- a/libtest/test.h +++ b/libtest/test.h @@ -227,6 +227,16 @@ do \ } \ } while (0) +#define test_false_with(A,B) \ +do \ +{ \ + if ((A)) { \ + fprintf(stderr, "\nAssertion failed at %s:%d: %s with %s\n", __FILE__, __LINE__, #A, (B));\ + create_core(); \ + return TEST_FAILURE; \ + } \ +} while (0) + #define test_strcmp(A,B) \ do \ { \ diff --git a/tests/mem_functions.c b/tests/mem_functions.c index 877a715c..ccd4b95b 100644 --- a/tests/mem_functions.c +++ b/tests/mem_functions.c @@ -6298,6 +6298,9 @@ test_st error_conditions[] ={ test_st parser_tests[] ={ + {"behavior", 0, (test_callback_fn)behavior_parser_test }, + {"distribtions", 0, (test_callback_fn)parser_distribution_test }, + {"hash", 0, (test_callback_fn)parser_hash_test }, {"server", 0, (test_callback_fn)server_test }, {"servers", 0, (test_callback_fn)servers_test }, {0, 0, (test_callback_fn)0} diff --git a/tests/parser.cc b/tests/parser.cc index 823970ed..da3c45d3 100644 --- a/tests/parser.cc +++ b/tests/parser.cc @@ -103,6 +103,89 @@ test_return_t servers_test(memcached_st *junk) memcached_servers_reset(memc); } + scanner_string_st bad_test_strings[]= { + { STRING_WITH_LEN("-servers=localhost:11221,localhost:11222,localhost:11223,localhost:11224,localhost:11225") }, + { STRING_WITH_LEN("-- servers=a.example.com:81,localhost:82,b.example.com") }, + { STRING_WITH_LEN("--servers=localhost80") }, + { NULL, 0} + }; + + for (scanner_string_st *ptr= bad_test_strings; ptr->size; ptr++) + { + memcached_return_t rc; + rc= memcached_parse_options(memc, ptr->c_ptr, ptr->size); + + test_false_with(rc == MEMCACHED_SUCCESS, ptr->c_ptr); + + memcached_server_fn callbacks[1]; + callbacks[0]= server_print_callback; + memcached_server_cursor(memc, callbacks, NULL, 1); + + memcached_servers_reset(memc); + } + + memcached_free(memc); + + return TEST_SUCCESS; +} + +test_return_t behavior_parser_test(memcached_st *junk) +{ + (void)junk; + return TEST_SUCCESS; +} + +test_return_t parser_hash_test(memcached_st *junk) +{ + (void)junk; + memcached_return_t rc; + memcached_st *memc; + memc= memcached_create(NULL); + + scanner_string_st test_strings[]= { + { STRING_WITH_LEN("--HASH=MD5") }, + { STRING_WITH_LEN("--HASH=CRC") }, + { STRING_WITH_LEN("--HASH=FNV1_64") }, + { STRING_WITH_LEN("--HASH=FNV1A_64") }, + { STRING_WITH_LEN("--HASH=FNV1_32") }, + { STRING_WITH_LEN("--HASH=FNV1A_32") }, + { STRING_WITH_LEN("--HASH=HSIEH") }, + { STRING_WITH_LEN("--HASH=MURMUR") }, + { STRING_WITH_LEN("--HASH=JENKINS") }, + { NULL, 0} + }; + + for (scanner_string_st *ptr= test_strings; ptr->size; ptr++) + { + rc= memcached_parse_options(memc, ptr->c_ptr, ptr->size); + test_true_got(rc == MEMCACHED_SUCCESS, ptr->c_ptr); + } + + memcached_free(memc); + + return TEST_SUCCESS; +} + +test_return_t parser_distribution_test(memcached_st *junk) +{ + (void)junk; + memcached_return_t rc; + memcached_st *memc; + memc= memcached_create(NULL); + + scanner_string_st test_strings[]= { + { STRING_WITH_LEN("--DISTRIBUTION=consistent") }, + { STRING_WITH_LEN("--DISTRIBUTION=random") }, + { STRING_WITH_LEN("--DISTRIBUTION=modula") }, + { NULL, 0} + }; + + for (scanner_string_st *ptr= test_strings; ptr->size; ptr++) + { + rc= memcached_parse_options(memc, ptr->c_ptr, ptr->size); + test_true_got(rc == MEMCACHED_SUCCESS, ptr->c_ptr); + } + memcached_free(memc); return TEST_SUCCESS; diff --git a/tests/parser.h b/tests/parser.h index 82d04031..08fe90cf 100644 --- a/tests/parser.h +++ b/tests/parser.h @@ -49,6 +49,15 @@ test_return_t server_test(memcached_st *memc); LIBTEST_INTERNAL_API test_return_t servers_test(memcached_st *memc); +LIBTEST_INTERNAL_API +test_return_t behavior_parser_test(memcached_st *junk); + +LIBTEST_INTERNAL_API +test_return_t parser_distribution_test(memcached_st *junk); + +LIBTEST_INTERNAL_API +test_return_t parser_hash_test(memcached_st *junk); + #ifdef __cplusplus } #endif