提交: 65142a1d8af111906f8c2f4fcc841cc5362977e3 - dogrun-ci (git)

docker based ci tool

Commit MetaInfo

• 修訂: 65142a1d8af111906f8c2f4fcc841cc5362977e3 (tree)
• 時間: 2019-06-06 02:19:38
• 作者: hylom <hylom@user...>
• Commiter: hylom

Log Message

core-server: fix log extractor

Change Summary

• modified: core-server/config.py
• modified: core-server/dockrun_pb2.py
• modified: core-server/server.py

差異

--- a/core-server/config.py
+++ b/core-server/config.py
@@ -5,6 +5,11 @@ config = {
             "auto_remove": True,
             "timeout": 60,
         },
+        "newslash": {
+            "container": "newslash-test",
+            "run_mode": "start",
+            "timeout": 300,
+        }
     },
 }
 

--- a/core-server/dockrun_pb2.py
+++ b/core-server/dockrun_pb2.py
@@ -19,13 +19,50 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   name='dockrun.proto',
   package='dockrun',
   syntax='proto3',
-  serialized_pb=_b('\n\rdockrun.proto\x12\x07\x64ockrun\"#\n\x0eRunTaskRequest\x12\x11\n\ttask_name\x18\x01 \x01(\t\"3\n\x0cRunTaskReply\x12\x12\n\nis_succeed\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t2F\n\x07\x44ockRun\x12;\n\x07RunTask\x12\x17.dockrun.RunTaskRequest\x1a\x15.dockrun.RunTaskReply\"\x00\x62\x06proto3')
+  serialized_pb=_b('\n\rdockrun.proto\x12\x07\x64ockrun\"\xd1\x01\n\x0eRunTaskRequest\x12\x11\n\ttask_name\x18\x01 \x01(\t\x12\x13\n\x0b\x63lient_name\x18\x02 \x01(\t\x12\x12\n\nuser_agent\x18\x03 \x01(\t\x12\x39\n\tparameter\x18\x04 \x03(\x0b\x32&.dockrun.RunTaskRequest.ParameterEntry\x12\x16\n\x0eparameter_type\x18\x05 \x01(\t\x1a\x30\n\x0eParameterEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"3\n\x0cRunTaskReply\x12\x12\n\nis_succeed\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t2F\n\x07\x44ockRun\x12;\n\x07RunTask\x12\x17.dockrun.RunTaskRequest\x1a\x15.dockrun.RunTaskReply\"\x00\x62\x06proto3')
 )
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
 
 
 
+_RUNTASKREQUEST_PARAMETERENTRY = _descriptor.Descriptor(
+  name='ParameterEntry',
+  full_name='dockrun.RunTaskRequest.ParameterEntry',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='key', full_name='dockrun.RunTaskRequest.ParameterEntry.key', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='value', full_name='dockrun.RunTaskRequest.ParameterEntry.value', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=188,
+  serialized_end=236,
+)
+
 _RUNTASKREQUEST = _descriptor.Descriptor(
   name='RunTaskRequest',
   full_name='dockrun.RunTaskRequest',
@@ -40,10 +77,38 @@ _RUNTASKREQUEST = _descriptor.Descriptor(
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
+    _descriptor.FieldDescriptor(
+      name='client_name', full_name='dockrun.RunTaskRequest.client_name', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='user_agent', full_name='dockrun.RunTaskRequest.user_agent', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='parameter', full_name='dockrun.RunTaskRequest.parameter', index=3,
+      number=4, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='parameter_type', full_name='dockrun.RunTaskRequest.parameter_type', index=4,
+      number=5, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
   ],
   extensions=[
   ],
-  nested_types=[],
+  nested_types=[_RUNTASKREQUEST_PARAMETERENTRY, ],
   enum_types=[
   ],
   options=None,
@@ -52,8 +117,8 @@ _RUNTASKREQUEST = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=26,
-  serialized_end=61,
+  serialized_start=27,
+  serialized_end=236,
 )
 
 
@@ -90,19 +155,29 @@ _RUNTASKREPLY = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=63,
-  serialized_end=114,
+  serialized_start=238,
+  serialized_end=289,
 )
 
+_RUNTASKREQUEST_PARAMETERENTRY.containing_type = _RUNTASKREQUEST
+_RUNTASKREQUEST.fields_by_name['parameter'].message_type = _RUNTASKREQUEST_PARAMETERENTRY
 DESCRIPTOR.message_types_by_name['RunTaskRequest'] = _RUNTASKREQUEST
 DESCRIPTOR.message_types_by_name['RunTaskReply'] = _RUNTASKREPLY
 
 RunTaskRequest = _reflection.GeneratedProtocolMessageType('RunTaskRequest', (_message.Message,), dict(
+
+  ParameterEntry = _reflection.GeneratedProtocolMessageType('ParameterEntry', (_message.Message,), dict(
+    DESCRIPTOR = _RUNTASKREQUEST_PARAMETERENTRY,
+    __module__ = 'dockrun_pb2'
+    # @@protoc_insertion_point(class_scope:dockrun.RunTaskRequest.ParameterEntry)
+    ))
+  ,
   DESCRIPTOR = _RUNTASKREQUEST,
   __module__ = 'dockrun_pb2'
   # @@protoc_insertion_point(class_scope:dockrun.RunTaskRequest)
   ))
 _sym_db.RegisterMessage(RunTaskRequest)
+_sym_db.RegisterMessage(RunTaskRequest.ParameterEntry)
 
 RunTaskReply = _reflection.GeneratedProtocolMessageType('RunTaskReply', (_message.Message,), dict(
   DESCRIPTOR = _RUNTASKREPLY,
@@ -112,6 +187,8 @@ RunTaskReply = _reflection.GeneratedProtocolMessageType('RunTaskReply', (_messag
 _sym_db.RegisterMessage(RunTaskReply)
 
 
+_RUNTASKREQUEST_PARAMETERENTRY.has_options = True
+_RUNTASKREQUEST_PARAMETERENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
 try:
   # THESE ELEMENTS WILL BE DEPRECATED.
   # Please use the generated *_pb2_grpc.py files instead.

--- a/core-server/server.py
+++ b/core-server/server.py
@@ -24,29 +24,47 @@ OUTPUT_DIR = './results'
 
 executor = concurrent.futures.ThreadPoolExecutor(max_workers=2)
 
-def run_task(task_name, task):
-    print("run task: '{}'".format(task_name))
+def run_task(param, task):
+    print("run task: '{}'".format(param["task_name"]))
 
     status = {
         "succeeded": False,
-        "container_status": "Running"
+        "container_status": "Running",
+        "user_agent": param["user_agent"],
+        "parameter": {},
+        "parameter_type": param["parameter_type"],
     };
-    reporter = FileReporter(task_name, status)
-    reporter.write_status()
+    for k in param["parameter"].keys():
+        status["parameter"][k] = param["parameter"][k]
 
-    image = task["image"]
-    client = docker.from_env()
-    container = client.containers.run(image=image, detach=True)
-    container.wait(timeout=task.get("timeout", 60))
+    reporter = FileReporter(param["task_name"], status)
+    reporter.write_status()
 
+    run_mode = task.get("run_mode", "run");
+    start_epoch = time.time()
+    
+    if run_mode == "run":
+        image = task["image"]
+        client = docker.from_env()
+        container = client.containers.run(image=image, detach=True)
+        container.wait(timeout=task.get("timeout", task.get("timeout", 60)))
+    elif run_mode == "start":
+        client = docker.from_env()
+        container = client.containers.get(task["container"])
+        container.start()
+        container.wait(timeout=task.get("timeout", task.get("timeout", 60)))
+        
     print("task done.")
 
-    lines = []
-    for line in container.logs(stream=True):
-        lines.append(line)
-        logs = b"".join(lines)
+    #lines = []
+    #for line in container.logs(stream=True):
+    #    lines.append(line)
+    #    logs = b"".join(lines)
+    logs = container.logs(since=start_epoch)
+
+    print("extract logs done.")
 
-    if task.get("auto_remove", True):
+    if run_mode == "run" and task.get("auto_remove", True):
         container.remove()
         print("container removed.")
 
@@ -56,7 +74,7 @@ def run_task(task_name, task):
     reporter.write_log()
 
     print("done.")
-    return
+    return True
 
 
 class FileReporter():
@@ -96,45 +114,37 @@ class FileReporter():
         return out_dir
 
     
+def _exec_done(future):
+    result = future.result()
+    print("task done.")
+    print(result)
+            
 class DockRun(DockRunServicer):
     def RunTask(self, req, context):
         tasks = cfg.get("tasks", {})
         task = tasks.get(req.task_name)
 
+        print("RunTask: {}".format(req.task_name))
+
         if not task:
             message = "task '{}' not found.".format(req.task_name)
+            print(message);
             return RunTaskReply(is_succeed=False, message=message)
 
         # generate process
-        #self._run_task(task)
-        executor.submit(run_task, req.task_name, task)
+        param = {
+            "task_name": req.task_name,
+            "client_name": req.client_name,
+            "user_agent": req.user_agent,
+            "parameter": req.parameter,
+            "parameter_type": req.parameter_type,
+        }
+        print("execute task...")
+        future = executor.submit(run_task, param, task)
+        future.add_done_callback(_exec_done)
 
         return RunTaskReply(is_succeed=True, message="")
 
-    def _run_task(self, task):
-        image = task["image"]
-        client = docker.from_env()
-        container = client.containers.run(image=image, detach=True)
-        container.wait(timeout=task.get("timeout", 60))
-
-        lines = []
-        for line in container.logs(stream=True):
-            lines.append(line)
-        logs = b"".join(lines)
-
-        if task.get("auto_remove", True):
-            container.remove()
-
-        report = {
-            "succeeded": False,
-            "status": None,
-        };
-
-        reporter = FileReporter(req.task_name, report, logs)
-        reporter.write()
-
-        return
-
 def main():
     server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
     add_DockRunServicer_to_server(