Kouhei Sutou
null+****@clear*****
Wed Nov 5 13:14:54 JST 2014
Kouhei Sutou 2014-11-05 13:14:54 +0900 (Wed, 05 Nov 2014) New Revision: 0aa6c9f447c7f582e6d8747aa96c7b5cf9678a24 https://github.com/groonga/fluent-plugin-groonga/commit/0aa6c9f447c7f582e6d8747aa96c7b5cf9678a24 Message: out: support table definition If there is the same name table, the table is renamed and recreated by the specified definition. Is it OK? TODO: Document me Added files: sample/store-apache.conf test/output/test_table_definition.rb Modified files: lib/fluent/plugin/out_groonga.rb Modified: lib/fluent/plugin/out_groonga.rb (+121 -4) =================================================================== --- lib/fluent/plugin/out_groonga.rb 2014-11-05 13:14:16 +0900 (e8c194a) +++ lib/fluent/plugin/out_groonga.rb 2014-11-05 13:14:54 +0900 (b0fd899) @@ -39,6 +39,17 @@ module Fluent end # alias is just for backward compatibility config_param :store_table, :string, :default => nil, :alias => :table + config_section :table, + :param_name => "tables", + :required => false, + :multi => true do |config| + config_param :name, :string + config_param :flags, :string, :default => nil + config_param :key_type, :string, :default => nil + config_param :default_tokenizer, :string, :default => nil + config_param :token_filters, :string, :default => nil + config_param :normalizer, :string, :default => nil + end def configure(conf) super @@ -46,12 +57,17 @@ module Fluent @client.configure(conf) @emitter = Emitter.new(@client, @store_table) + + @tables =****@table***** do |table| + end end def start super @client.start @emitter.start + tables_creator = TablesCreator.new(@client, @tables) + tables_creator.create end def shutdown @@ -78,6 +94,111 @@ module Fluent end end + class TableDefinition + def initialize(raw) + @raw = raw + end + + def name + @raw[:name] + end + + def flags + parse_flags(@raw[:flags] || "TABLE_NO_KEY") + end + + def key_type + @raw[:key_type] + end + + def default_tokenizer + @raw[:default_tokenizer] + end + + def token_filters + parse_items(@raw[:token_filters] || "") + end + + def normalizer + @raw[:normalizer] + end + + def have_difference?(table) + return true if table.name != name + + table_flags = (parse_flags(table.flags) - ["PERSISTENT"]) + return true if table_flags.sort != flags.sort + + return true if table.domain != key_type + + return true if table.default_tokenizer != default_tokenizer + + # TODO + # return true if table.token_filters.sort != token_filters.sort + + return true if table.normalizer != normalizer + + false + end + + def to_create_arguments + arguments = { + "name" => name, + "flags" => flags.join("|"), + "key_type" => key_type, + "default_tokenizer" => default_tokenizer, + # TODO + # "token_filters" => token_filters.join("|"), + "normalizer" => normalizer, + } + arguments.keys.each do |key| + value = arguments[key] + arguments.delete(key) if value.nil? or value.empty? + end + arguments + end + + private + def parse_flags(flags) + if flags.is_a?(Array) + flags + else + flags.strip.split(/\s*\|\s*/) + end + end + + def parse_items(items) + if items.is_a?(Array) + items + else + items.strip.split(/\s*,\s*/) + end + end + end + + class TablesCreator + def initialize(client, definitions) + @client = client + @definitions = definitions + end + + def create + table_list =****@clien*****("table_list") + @definitions.each do |definition| + existing_table = table_list.find do |table| + table.name == definition.name + end + if existing_table + next unless definition.have_difference?(existing_table) + # TODO: Is it OK? + @client.execute("table_remove", "name" => definition.name) + end + + @client.execute("table_create", definition.to_create_arguments) + end + end + end + class Schema def initialize(client, table_name) @client = client @@ -86,10 +207,6 @@ module Fluent @columns = nil end - def populate - # TODO - end - def update(records) ensure_table ensure_columns Added: sample/store-apache.conf (+118 -0) 100644 =================================================================== --- /dev/null +++ sample/store-apache.conf 2014-11-05 13:14:54 +0900 (a118082) @@ -0,0 +1,118 @@ +<source> + type forward +</source> + +<source> + type tail + path /var/log/apache2/access.log + pos_file /tmp/apache_access.pos + tag apache.raw.log.apache.access + format apache2 + read_from_head "#{ENV['FLUENT_PLUGIN_GROONGA_DEBUG'] == 'yes'}" +</source> + +<match apache.**> + type record_reformer + enable_ruby false + + tag ${tag_suffix[1]} + + <record> + remote ${host} + </record> +</match> + +<match raw.log.**> + type record_reformer + enable_ruby false + + tag ${tag_suffix[1]} + + <record> + host ${hostname} + type ${tag_suffix[2]} + timestamp ${time} + </record> +</match> + +<match log.*> + type groonga + store_table Logs + + protocol http + host 127.0.0.1 + + buffer_type file + buffer_path /tmp/buffer + flush_interval 1 + + <table> + name Terms + flags TABLE_PAT_KEY + key_type ShortText + default_tokenizer TokenBigram + normalizer NormalizerAuto + </table> + + <table> + name Codes + flags TABLE_PAT_KEY + key_type Int32 + </table> + + <table> + name Hosts + flags TABLE_PAT_KEY + key_type ShortText + normalizer NormalizerAuto + </table> + + <table> + name URLs + flags TABLE_PAT_KEY + key_type ShortText + </table> + + <table> + name Paths + flags TABLE_PAT_KEY + key_type ShortText + </table> + + <table> + name UserAgents + flags TABLE_PAT_KEY + key_type ShortText + </table> + + <table> + name Methods + flags TABLE_HASH_KEY + key_type ShortText + normalizer NormalizerAuto + </table> + + <table> + name Remotes + flags TABLE_PAT_KEY + key_type ShortText + </table> + + <table> + name Sizes + flags TABLE_PAT_KEY + key_type Int32 + </table> + + <table> + name Timestamps + flags TABLE_PAT_KEY + key_type Time + </table> + + <table> + name Types + flags TABLE_PAT_KEY + key_type ShortText + </table> +</match> Added: test/output/test_table_definition.rb (+167 -0) 100644 =================================================================== --- /dev/null +++ test/output/test_table_definition.rb 2014-11-05 13:14:54 +0900 (4423b5b) @@ -0,0 +1,167 @@ +# Copyright (C) 2014 Kouhei Sutou <kou �� clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +require "fluent/plugin/out_groonga" + +class OutputTypeTableDefinitionTest < Test::Unit::TestCase + def definition(raw={}) + Fluent::GroongaOutput::TableDefinition.new(raw) + end + + sub_test_case "readers" do + sub_test_case "\#name" do + test "specified" do + assert_equal("Tags", definition(:name => "Tags").name) + end + end + + sub_test_case "\#flags" do + test "default" do + assert_equal(["TABLE_NO_KEY"], + definition.flags) + end + + test "one" do + assert_equal(["TABLE_PAT_KEY"], + definition(:flags => "TABLE_PAT_KEY").flags) + end + end + + sub_test_case "\#key_type" do + test "default" do + assert_nil(definition.key_type) + end + + test "specified" do + assert_equal("ShortText", + definition(:key_type => "ShortText").key_type) + end + end + + sub_test_case "\#default_tokenizer" do + def read_default_tokenizer(input) + definition(:default_tokenizer => input).default_tokenizer + end + + test "default" do + assert_nil(definition.default_tokenizer) + end + + test "specified" do + assert_equal("TokenBigram", + read_default_tokenizer("TokenBigram")) + end + end + + sub_test_case "\#token_filters" do + def read_token_filters(input) + definition(:token_filters => input).token_filters + end + + test "default" do + assert_equal([], definition.token_filters) + end + + test "one" do + assert_equal(["TokenFilterStem"], + read_token_filters("TokenFilterStem")) + end + + test "multiple" do + assert_equal(["TokenFilterStem", "TokenFilterStopWord"], + read_token_filters("TokenFilterStem,TokenFilterStopWord")) + end + end + + sub_test_case "\#normalizer" do + def read_normalizer(input) + definition(:normalizer => input).normalizer + end + + test "default" do + assert_nil(definition.normalizer) + end + + test "specified" do + assert_equal("NormalizerAuto", + read_normalizer("NormalizerAuto")) + end + end + end + + sub_test_case "\#have_difference?" do + def setup + @existing_table = Groonga::Client::Response::TableList::Table.new + @existing_table.id = 260 + @existing_table.name = "Paths" + @existing_table.path = "/var/lib/groonga/db/db.0000104" + @existing_table.flags = "TABLE_PAT_KEY|PERSISTENT" + @existing_table.domain = "ShortText" + @existing_table.range = nil + @existing_table.default_tokenizer = nil + @existing_table.normalizer = nil + end + + def have_difference?(raw={}) + default_raw = { + :name => @existing_table.name, + :flags => @existing_table.flags.gsub(/\|PERSISTENT/, ""), + :key_type => @existing_table.domain, + :default_tokenizer => @existing_table.default_tokenizer, + :normalizer => @existing_table.normalizer, + } + raw = default_raw.merge(raw) + definition(raw).have_difference?(@existing_table) + end + + test "no difference" do + assert do + not have_difference? + end + end + + sub_test_case "difference" do + test "name" do + assert do + have_difference?(:name => "Difference") + end + end + + test "flags" do + assert do + have_difference?(:flags => "TABLE_NO_KEY") + end + end + + test "key_type" do + assert do + have_difference?(:key_type => "UInt32") + end + end + + test "default_tokenizer" do + assert do + have_difference?(:default_tokenizer => "TokenBigram") + end + end + + test "normalizer" do + assert do + have_difference?(:normalizer => "NormalizerAuto") + end + end + end + end +end -------------- next part -------------- HTML����������������������������...下載
