Kouhei Sutou
null+****@clear*****
Mon Nov 17 22:40:45 JST 2014
Kouhei Sutou 2014-11-17 22:40:45 +0900 (Mon, 17 Nov 2014) New Revision: e2b455172da18147ccc7a067137aae2555126caf https://github.com/groonga/groonga/commit/e2b455172da18147ccc7a067137aae2555126caf Message: Extract grn_token declarations Added files: include/groonga/token.h Modified files: include/groonga/Makefile.am include/groonga/tokenizer.h Modified: include/groonga/Makefile.am (+1 -0) =================================================================== --- include/groonga/Makefile.am 2014-11-17 22:35:20 +0900 (7e3e9c0) +++ include/groonga/Makefile.am 2014-11-17 22:40:45 +0900 (cd131fc) @@ -4,6 +4,7 @@ groonga_include_HEADERS = \ groonga.h \ ii.h \ plugin.h \ + token.h \ tokenizer.h \ token_filter.h \ nfkc.h \ Added: include/groonga/token.h (+118 -0) 100644 =================================================================== --- /dev/null +++ include/groonga/token.h 2014-11-17 22:40:45 +0900 (1ba9307) @@ -0,0 +1,118 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2014 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ +#ifndef GROONGA_TOKEN_H +#define GROONGA_TOKEN_H + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/* + grn_token_mode describes propose for tokenization. + + `GRN_TOKEN_GET`: Tokenization for search. + + `GRN_TOKEN_ADD`: Tokenization for adding token to index. + + `GRN_TOKEN_DEL`: Tokenization for deleting token from index. + + @since 4.0.7 + */ +typedef enum { + GRN_TOKEN_GET = 0, + GRN_TOKEN_ADD, + GRN_TOKEN_DEL +} grn_token_mode; + +/* + * grn_token_status is a flag set for tokenizer status codes. + * If a document or query contains no tokens, push an empty string with + * GRN_TOKEN_LAST as a token. + * + * @since 4.0.7 + */ +typedef unsigned int grn_token_status; + +/* + * GRN_TOKEN_CONTINUE means that the next token is not the last one. + * + * @since 4.0.7 + */ +#define GRN_TOKEN_CONTINUE (0) +/* + * GRN_TOKEN_LAST means that the next token is the last one. + * + * @since 4.0.7 + */ +#define GRN_TOKEN_LAST (0x01L<<0) +/* + * GRN_TOKEN_OVERLAP means that ... + * + * @since 4.0.7 + */ +#define GRN_TOKEN_OVERLAP (0x01L<<1) +/* + * GRN_TOKEN_UNMATURED means that ... + * + * @since 4.0.7 + */ +#define GRN_TOKEN_UNMATURED (0x01L<<2) +/* + * GRN_TOKEN_REACH_END means that ... + * + * @since 4.0.7 + */ +#define GRN_TOKEN_REACH_END (0x01L<<3) +/* + * GRN_TOKEN_SKIP means that the token is skipped + * + * @since 4.0.7 + */ +#define GRN_TOKEN_SKIP (0x01L<<4) +/* + * GRN_TOKEN_SKIP_WITH_POSITION means that the token and postion is skipped + * + * @since 4.0.7 + */ +#define GRN_TOKEN_SKIP_WITH_POSITION (0x01L<<5) +/* + * GRN_TOKEN_FORCE_PREIX that the token is used common prefix search + * + * @since 4.0.7 + */ +#define GRN_TOKEN_FORCE_PREFIX (0x01L<<6) + +typedef struct _grn_token grn_token; + +GRN_PLUGIN_EXPORT grn_obj *grn_token_get_data(grn_ctx *ctx, + grn_token *token); +GRN_PLUGIN_EXPORT grn_rc grn_token_set_data(grn_ctx *ctx, + grn_token *token, + const char *str_ptr, + int str_length); +GRN_PLUGIN_EXPORT grn_token_status grn_token_get_status(grn_ctx *ctx, + grn_token *token); +GRN_PLUGIN_EXPORT grn_rc grn_token_set_status(grn_ctx *ctx, + grn_token *token, + grn_token_status status); + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* GROONGA_TOKEN_H */ Modified: include/groonga/tokenizer.h (+1 -92) =================================================================== --- include/groonga/tokenizer.h 2014-11-17 22:35:20 +0900 (3b4a483) +++ include/groonga/tokenizer.h 2014-11-17 22:40:45 +0900 (e5e279c) @@ -18,9 +18,8 @@ #ifndef GROONGA_TOKENIZER_H #define GROONGA_TOKENIZER_H -#include <stddef.h> - #include <groonga/plugin.h> +#include <groonga/token.h> #ifdef __cplusplus extern "C" { @@ -30,23 +29,6 @@ extern "C" { #define GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8_LEN 3 /* - grn_token_mode describes propose for tokenization. - - `GRN_TOKEN_GET`: Tokenization for search. - - `GRN_TOKEN_ADD`: Tokenization for adding token to index. - - `GRN_TOKEN_DEL`: Tokenization for deleting token from index. - - @since 4.0.7 - */ -typedef enum { - GRN_TOKEN_GET = 0, - GRN_TOKEN_ADD, - GRN_TOKEN_DEL -} grn_token_mode; - -/* grn_tokenizer_charlen() returns the length (#bytes) of the first character in the string specified by `str_ptr' and `str_length'. If the starting bytes are invalid as a character, grn_tokenizer_charlen() returns 0. See @@ -163,64 +145,6 @@ GRN_PLUGIN_EXPORT void grn_tokenizer_token_init(grn_ctx *ctx, grn_tokenizer_toke GRN_PLUGIN_EXPORT void grn_tokenizer_token_fin(grn_ctx *ctx, grn_tokenizer_token *token); /* - * grn_token_status is a flag set for tokenizer status codes. - * If a document or query contains no tokens, push an empty string with - * GRN_TOKEN_LAST as a token. - * - * @since 4.0.7 - */ -typedef unsigned int grn_token_status; - -/* - * GRN_TOKEN_CONTINUE means that the next token is not the last one. - * - * @since 4.0.7 - */ -#define GRN_TOKEN_CONTINUE (0) -/* - * GRN_TOKEN_LAST means that the next token is the last one. - * - * @since 4.0.7 - */ -#define GRN_TOKEN_LAST (0x01L<<0) -/* - * GRN_TOKEN_OVERLAP means that ... - * - * @since 4.0.7 - */ -#define GRN_TOKEN_OVERLAP (0x01L<<1) -/* - * GRN_TOKEN_UNMATURED means that ... - * - * @since 4.0.7 - */ -#define GRN_TOKEN_UNMATURED (0x01L<<2) -/* - * GRN_TOKEN_REACH_END means that ... - * - * @since 4.0.7 - */ -#define GRN_TOKEN_REACH_END (0x01L<<3) -/* - * GRN_TOKEN_SKIP means that the token is skipped - * - * @since 4.0.7 - */ -#define GRN_TOKEN_SKIP (0x01L<<4) -/* - * GRN_TOKEN_SKIP_WITH_POSITION means that the token and postion is skipped - * - * @since 4.0.7 - */ -#define GRN_TOKEN_SKIP_WITH_POSITION (0x01L<<5) -/* - * GRN_TOKEN_FORCE_PREIX that the token is used common prefix search - * - * @since 4.0.7 - */ -#define GRN_TOKEN_FORCE_PREFIX (0x01L<<6) - -/* * grn_tokenizer_status is a flag set for tokenizer status codes. * If a document or query contains no tokens, push an empty string with * GRN_TOKENIZER_TOKEN_LAST as a token. @@ -287,21 +211,6 @@ typedef grn_token_status grn_tokenizer_status; #define GRN_TOKENIZER_CONTINUE GRN_TOKENIZER_TOKEN_CONTINUE #define GRN_TOKENIZER_LAST GRN_TOKENIZER_TOKEN_LAST -typedef struct _grn_token grn_token; - -GRN_PLUGIN_EXPORT grn_obj *grn_token_get_data(grn_ctx *ctx, - grn_token *token); -GRN_PLUGIN_EXPORT grn_rc grn_token_set_data(grn_ctx *ctx, - grn_token *token, - const char *str_ptr, - int str_length); -GRN_PLUGIN_EXPORT grn_tokenizer_status grn_token_get_status(grn_ctx *ctx, - grn_token *token); -GRN_PLUGIN_EXPORT grn_rc grn_token_set_status(grn_ctx *ctx, - grn_token *token, - grn_token_status status); - - /* grn_tokenizer_token_push() pushes the next token into `token'. Note that grn_tokenizer_token_push() does not make a copy of the given string. This -------------- next part -------------- HTML����������������������������...下載
