[Groonga-commit] groonga/fluent-plugin-groonga at 0aa6c9f [master] out: support table definition

Back to archive index

Kouhei Sutou null+****@clear*****
Wed Nov 5 13:14:54 JST 2014


Kouhei Sutou	2014-11-05 13:14:54 +0900 (Wed, 05 Nov 2014)

  New Revision: 0aa6c9f447c7f582e6d8747aa96c7b5cf9678a24
  https://github.com/groonga/fluent-plugin-groonga/commit/0aa6c9f447c7f582e6d8747aa96c7b5cf9678a24

  Message:
    out: support table definition
    
    If there is the same name table, the table is renamed and recreated by
    the specified definition. Is it OK?
    
    TODO: Document me

  Added files:
    sample/store-apache.conf
    test/output/test_table_definition.rb
  Modified files:
    lib/fluent/plugin/out_groonga.rb

  Modified: lib/fluent/plugin/out_groonga.rb (+121 -4)
===================================================================
--- lib/fluent/plugin/out_groonga.rb    2014-11-05 13:14:16 +0900 (e8c194a)
+++ lib/fluent/plugin/out_groonga.rb    2014-11-05 13:14:54 +0900 (b0fd899)
@@ -39,6 +39,17 @@ module Fluent
     end
     # alias is just for backward compatibility
     config_param :store_table, :string, :default => nil, :alias => :table
+    config_section :table,
+                   :param_name => "tables",
+                   :required => false,
+                   :multi => true do |config|
+      config_param :name, :string
+      config_param :flags, :string, :default => nil
+      config_param :key_type, :string, :default => nil
+      config_param :default_tokenizer, :string, :default => nil
+      config_param :token_filters, :string, :default => nil
+      config_param :normalizer, :string, :default => nil
+    end
 
     def configure(conf)
       super
@@ -46,12 +57,17 @@ module Fluent
       @client.configure(conf)
 
       @emitter = Emitter.new(@client, @store_table)
+
+      @tables =****@table***** do |table|
+      end
     end
 
     def start
       super
       @client.start
       @emitter.start
+      tables_creator = TablesCreator.new(@client, @tables)
+      tables_creator.create
     end
 
     def shutdown
@@ -78,6 +94,111 @@ module Fluent
       end
     end
 
+    class TableDefinition
+      def initialize(raw)
+        @raw = raw
+      end
+
+      def name
+        @raw[:name]
+      end
+
+      def flags
+        parse_flags(@raw[:flags] || "TABLE_NO_KEY")
+      end
+
+      def key_type
+        @raw[:key_type]
+      end
+
+      def default_tokenizer
+        @raw[:default_tokenizer]
+      end
+
+      def token_filters
+        parse_items(@raw[:token_filters] || "")
+      end
+
+      def normalizer
+        @raw[:normalizer]
+      end
+
+      def have_difference?(table)
+        return true if table.name != name
+
+        table_flags = (parse_flags(table.flags) - ["PERSISTENT"])
+        return true if table_flags.sort != flags.sort
+
+        return true if table.domain != key_type
+
+        return true if table.default_tokenizer != default_tokenizer
+
+        # TODO
+        # return true if table.token_filters.sort != token_filters.sort
+
+        return true if table.normalizer != normalizer
+
+        false
+      end
+
+      def to_create_arguments
+        arguments = {
+          "name" => name,
+          "flags" => flags.join("|"),
+          "key_type" => key_type,
+          "default_tokenizer" => default_tokenizer,
+          # TODO
+          # "token_filters" => token_filters.join("|"),
+          "normalizer" => normalizer,
+        }
+        arguments.keys.each do |key|
+          value = arguments[key]
+          arguments.delete(key) if value.nil? or value.empty?
+        end
+        arguments
+      end
+
+      private
+      def parse_flags(flags)
+        if flags.is_a?(Array)
+          flags
+        else
+          flags.strip.split(/\s*\|\s*/)
+        end
+      end
+
+      def parse_items(items)
+        if items.is_a?(Array)
+          items
+        else
+          items.strip.split(/\s*,\s*/)
+        end
+      end
+    end
+
+    class TablesCreator
+      def initialize(client, definitions)
+        @client = client
+        @definitions = definitions
+      end
+
+      def create
+        table_list =****@clien*****("table_list")
+        @definitions.each do |definition|
+          existing_table = table_list.find do |table|
+            table.name == definition.name
+          end
+          if existing_table
+            next unless definition.have_difference?(existing_table)
+            # TODO: Is it OK?
+            @client.execute("table_remove", "name" => definition.name)
+          end
+
+          @client.execute("table_create", definition.to_create_arguments)
+        end
+      end
+    end
+
     class Schema
       def initialize(client, table_name)
         @client = client
@@ -86,10 +207,6 @@ module Fluent
         @columns = nil
       end
 
-      def populate
-        # TODO
-      end
-
       def update(records)
         ensure_table
         ensure_columns

  Added: sample/store-apache.conf (+118 -0) 100644
===================================================================
--- /dev/null
+++ sample/store-apache.conf    2014-11-05 13:14:54 +0900 (a118082)
@@ -0,0 +1,118 @@
+<source>
+  type forward
+</source>
+
+<source>
+  type tail
+  path /var/log/apache2/access.log
+  pos_file /tmp/apache_access.pos
+  tag apache.raw.log.apache.access
+  format apache2
+  read_from_head "#{ENV['FLUENT_PLUGIN_GROONGA_DEBUG'] == 'yes'}"
+</source>
+
+<match apache.**>
+  type record_reformer
+  enable_ruby false
+
+  tag ${tag_suffix[1]}
+
+  <record>
+    remote ${host}
+  </record>
+</match>
+
+<match raw.log.**>
+  type record_reformer
+  enable_ruby false
+
+  tag ${tag_suffix[1]}
+
+  <record>
+    host ${hostname}
+    type ${tag_suffix[2]}
+    timestamp ${time}
+  </record>
+</match>
+
+<match log.*>
+  type groonga
+  store_table Logs
+
+  protocol http
+  host 127.0.0.1
+
+  buffer_type file
+  buffer_path /tmp/buffer
+  flush_interval 1
+
+  <table>
+    name Terms
+    flags TABLE_PAT_KEY
+    key_type ShortText
+    default_tokenizer TokenBigram
+    normalizer NormalizerAuto
+  </table>
+
+  <table>
+    name Codes
+    flags TABLE_PAT_KEY
+    key_type Int32
+  </table>
+
+  <table>
+    name Hosts
+    flags TABLE_PAT_KEY
+    key_type ShortText
+    normalizer NormalizerAuto
+  </table>
+
+  <table>
+    name URLs
+    flags TABLE_PAT_KEY
+    key_type ShortText
+  </table>
+
+  <table>
+    name Paths
+    flags TABLE_PAT_KEY
+    key_type ShortText
+  </table>
+
+  <table>
+    name UserAgents
+    flags TABLE_PAT_KEY
+    key_type ShortText
+  </table>
+
+  <table>
+    name Methods
+    flags TABLE_HASH_KEY
+    key_type ShortText
+    normalizer NormalizerAuto
+  </table>
+
+  <table>
+    name Remotes
+    flags TABLE_PAT_KEY
+    key_type ShortText
+  </table>
+
+  <table>
+    name Sizes
+    flags TABLE_PAT_KEY
+    key_type Int32
+  </table>
+
+  <table>
+    name Timestamps
+    flags TABLE_PAT_KEY
+    key_type Time
+  </table>
+
+  <table>
+    name Types
+    flags TABLE_PAT_KEY
+    key_type ShortText
+  </table>
+</match>

  Added: test/output/test_table_definition.rb (+167 -0) 100644
===================================================================
--- /dev/null
+++ test/output/test_table_definition.rb    2014-11-05 13:14:54 +0900 (4423b5b)
@@ -0,0 +1,167 @@
+# Copyright (C) 2014  Kouhei Sutou <kou �� clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "fluent/plugin/out_groonga"
+
+class OutputTypeTableDefinitionTest < Test::Unit::TestCase
+  def definition(raw={})
+    Fluent::GroongaOutput::TableDefinition.new(raw)
+  end
+
+  sub_test_case "readers" do
+    sub_test_case "\#name" do
+      test "specified" do
+        assert_equal("Tags", definition(:name => "Tags").name)
+      end
+    end
+
+    sub_test_case "\#flags" do
+      test "default" do
+        assert_equal(["TABLE_NO_KEY"],
+                     definition.flags)
+      end
+
+      test "one" do
+        assert_equal(["TABLE_PAT_KEY"],
+                     definition(:flags => "TABLE_PAT_KEY").flags)
+      end
+    end
+
+    sub_test_case "\#key_type" do
+      test "default" do
+        assert_nil(definition.key_type)
+      end
+
+      test "specified" do
+        assert_equal("ShortText",
+                     definition(:key_type => "ShortText").key_type)
+      end
+    end
+
+    sub_test_case "\#default_tokenizer" do
+      def read_default_tokenizer(input)
+        definition(:default_tokenizer => input).default_tokenizer
+      end
+
+      test "default" do
+        assert_nil(definition.default_tokenizer)
+      end
+
+      test "specified" do
+        assert_equal("TokenBigram",
+                     read_default_tokenizer("TokenBigram"))
+      end
+    end
+
+    sub_test_case "\#token_filters" do
+      def read_token_filters(input)
+        definition(:token_filters => input).token_filters
+      end
+
+      test "default" do
+        assert_equal([], definition.token_filters)
+      end
+
+      test "one" do
+        assert_equal(["TokenFilterStem"],
+                      read_token_filters("TokenFilterStem"))
+      end
+
+      test "multiple" do
+        assert_equal(["TokenFilterStem", "TokenFilterStopWord"],
+                      read_token_filters("TokenFilterStem,TokenFilterStopWord"))
+      end
+    end
+
+    sub_test_case "\#normalizer" do
+      def read_normalizer(input)
+        definition(:normalizer => input).normalizer
+      end
+
+      test "default" do
+        assert_nil(definition.normalizer)
+      end
+
+      test "specified" do
+        assert_equal("NormalizerAuto",
+                     read_normalizer("NormalizerAuto"))
+      end
+    end
+  end
+
+  sub_test_case "\#have_difference?" do
+    def setup
+      @existing_table = Groonga::Client::Response::TableList::Table.new
+      @existing_table.id = 260
+      @existing_table.name = "Paths"
+      @existing_table.path = "/var/lib/groonga/db/db.0000104"
+      @existing_table.flags = "TABLE_PAT_KEY|PERSISTENT"
+      @existing_table.domain = "ShortText"
+      @existing_table.range = nil
+      @existing_table.default_tokenizer = nil
+      @existing_table.normalizer = nil
+    end
+
+    def have_difference?(raw={})
+      default_raw = {
+        :name              => @existing_table.name,
+        :flags             => @existing_table.flags.gsub(/\|PERSISTENT/, ""),
+        :key_type          => @existing_table.domain,
+        :default_tokenizer => @existing_table.default_tokenizer,
+        :normalizer        => @existing_table.normalizer,
+      }
+      raw = default_raw.merge(raw)
+      definition(raw).have_difference?(@existing_table)
+    end
+
+    test "no difference" do
+      assert do
+        not have_difference?
+      end
+    end
+
+    sub_test_case "difference" do
+      test "name" do
+        assert do
+          have_difference?(:name => "Difference")
+        end
+      end
+
+      test "flags" do
+        assert do
+          have_difference?(:flags => "TABLE_NO_KEY")
+        end
+      end
+
+      test "key_type" do
+        assert do
+          have_difference?(:key_type => "UInt32")
+        end
+      end
+
+      test "default_tokenizer" do
+        assert do
+          have_difference?(:default_tokenizer => "TokenBigram")
+        end
+      end
+
+      test "normalizer" do
+        assert do
+          have_difference?(:normalizer => "NormalizerAuto")
+        end
+      end
+    end
+  end
+end
-------------- next part --------------
HTML����������������������������...
下載 



More information about the Groonga-commit mailing list
Back to archive index