提交: 54ef2ab8a19e3259303e019dc792a54f90a4fec8 - CCastle2 (hg)

Castle: The best Real-Time/Embedded/HighTech language EVER. Attempt 2

Commit MetaInfo

• 修訂: 54ef2ab8a19e3259303e019dc792a54f90a4fec8 (tree)
• 時間: 2022-03-07 02:45:19
• 作者: Albert Mietus < albert AT mietus DOT nl >
• Commiter: Albert Mietus < albert AT mietus DOT nl >

Log Message

started with generic Reader classes -- See PEGReader

Change Summary

• modified: castle/readers/parser/grammar.py
• modified: demos/ThinOnion/1rule.peg
• modified: demos/ThinOnion/demo1.py
• modified: pytst/readers/parser/d1_parse_tree/test_1_regexp.py
• modified: pytst/readers/parser/d1_parse_tree/test_9_file.py
• modified: pytst/readers/parser/d2_ast/test_1_term.py
• modified: pytst/readers/parser/grammar.peg
• add: castle/readers/general/__init__.py
• add: castle/readers/general/file.py
• add: castle/readers/parser/__init__.py
• add: pytst/readers/parser/general/test_file.py

差異

diff -r 76f7a4c5a47d -r 54ef2ab8a19e castle/readers/general/file.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/castle/readers/general/file.py	Sun Mar 06 18:45:19 2022 +0100
@@ -0,0 +1,34 @@
+""" General File-Reader support"""
+
+import logging; logger = logging.getLogger(__name__)
+
+
+from pathlib import Path
+import os
+
+
+def _get_file_dirPath(file=None):
+    if file is None: file=__file__
+    path_to_current_test = Path(os.path.realpath(file))
+    path_to_current_dir = path_to_current_test.parent
+    return path_to_current_dir
+
+
+
+class BaseReader():
+
+    def __init__(self, *, read_dirs: list[str], **kwargs):
+        if isinstance(read_dirs, str): read_dirs=[read_dirs] #Always a list
+        self.read_dirs = [ _get_file_dirPath() / d for d in read_dirs]
+        super().__init__(**kwargs)
+
+    def _read(self, filename) ->str:
+        for d in self.read_dirs:
+            if (d / filename).exists():
+                break
+        with (d / filename).open() as f:
+            logger.debug(f'Reading file: >>{f.name}<<')
+            txt = f.read()
+        return txt
+
+

diff -r 76f7a4c5a47d -r 54ef2ab8a19e castle/readers/parser/__init__.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/castle/readers/parser/__init__.py	Sun Mar 06 18:45:19 2022 +0100
@@ -0,0 +1,60 @@
+"""The  PEG :py:mod:`castle.readers.parser` module is based on Arpeggio
+
+.. seealso::
+
+   * **PEG** *(Parsing Expression Grammar*) https://en.wikipedia.org/wiki/Parsing_expression_grammar
+   * **Arpeggio** https://textx.github.io/Arpeggio
+
+"""
+
+import logging; logger = logging.getLogger(__name__)
+
+from typing import Callable
+
+import arpeggio
+from castle.readers.general.file import BaseReader
+
+
+class PEGReader(BaseReader):
+    """File-Reader to read & parse files, using a PEG parser.
+
+    When creating a :py:class:`PEGReader` a default value for the `language_def` & `comment_def` can be set (and/or can
+    be set when reading a file -- see py:func`parse`). This language_def & comment_def should defined using *Arpeggio*.
+
+    Also, a :py:class:`visitor` (class) should be specified. (this can only be done once).
+
+    When creating a class the search-path `read_dirs` (to look for file-to-be-parsed) can be set.
+    """
+
+    def __init__(self, *, read_dirs: list[str]=[],
+                 language_def=None, comment_def=None, visitor:Callable=None,
+                 **kwargs):
+        super().__init__(read_dirs=read_dirs, **kwargs)
+        if visitor is None:
+            raise ValueError("visitor is a mandatory parameter")
+        self._visitor = visitor
+        self.default_language_def = language_def
+        self.default_comment_def = comment_def
+
+
+    def parse(self,  filename:str, *, language_def=None, comment_def=None):
+        """Read & Parse a file"""
+        if not language_def: language_def = self.default_language_def
+        if not comment_def: comment_def = self.default_comment_def
+        txt = self._read(filename)
+        ast = self._do_parse(txt, language_def, comment_def)
+        return ast
+
+
+    def _do_parse(self, txt, language_def, comment_def):
+        parser = arpeggio.ParserPython(language_def=language_def, comment_def=comment_def)
+
+        pt = parser.parse(txt)
+        logger.info(f"Reader:_do_parse::\t parse_tree: start={pt.position} end={pt.position_end}; len(txt)={len(txt)}")
+
+        ast = arpeggio.visit_parse_tree(pt, self._visitor)
+        logger.debug(f"Reader:_do_parse::\t ast: start={ast.position} end={ast.position_end} -- not counting comments.")
+
+        return ast
+
+

diff -r 76f7a4c5a47d -r 54ef2ab8a19e castle/readers/parser/grammar.py
--- a/castle/readers/parser/grammar.py	Tue Mar 01 21:09:36 2022 +0100
+++ b/castle/readers/parser/grammar.py	Sun Mar 06 18:45:19 2022 +0100
@@ -58,6 +58,6 @@
 REs3 = _(r"[rR]'''")
 REd3 = _(r'[rR]"""')
 
-def comment():		return "//", _(".*\n")
+def comment():		return ["#", '//'], _(".*\n")
 
 

diff -r 76f7a4c5a47d -r 54ef2ab8a19e demos/ThinOnion/1rule.peg
--- a/demos/ThinOnion/1rule.peg	Tue Mar 01 21:09:36 2022 +0100
+++ b/demos/ThinOnion/1rule.peg	Sun Mar 06 18:45:19 2022 +0100
@@ -1,4 +1,11 @@
-//EXPECT:
-//def my_rule(): return crossref_1, "string", crossref_1
+peg_grammar	<-  rules EOF ;
+rules		<-  (parse_rule | setting)+ ;
+parse_rule	<-  rule_name '<-' expression ';' ;
 
-my_rule	<-  crossref_1 "string" crossref_2 ;
\ No newline at end of file
+expression	<-  sequence op_alternative ;
+sequence        <-  single_expr+ ;
+single_expr	<-  ( rule_crossref | term | group | predicate )  op_quantity ;
+
+op_alternative  <-  ( '|' expression )? ;
+op_quantity     <-  ( '?' | '*' | '+' | '\#' )? ;
+

diff -r 76f7a4c5a47d -r 54ef2ab8a19e demos/ThinOnion/demo1.py
--- a/demos/ThinOnion/demo1.py	Tue Mar 01 21:09:36 2022 +0100
+++ b/demos/ThinOnion/demo1.py	Sun Mar 06 18:45:19 2022 +0100
@@ -8,14 +8,6 @@
 import jinja2
 
 
-def _get_file_dirPath():
-    from pathlib import Path
-    import os
-    path_to_current_test = Path(os.path.realpath(__file__))
-    path_to_current_dir = path_to_current_test.parent
-    return path_to_current_dir
-
-
 class Reader():
 
     def __init__(self, read_dirs: list[str]):

diff -r 76f7a4c5a47d -r 54ef2ab8a19e pytst/readers/parser/d1_parse_tree/test_1_regexp.py
--- a/pytst/readers/parser/d1_parse_tree/test_1_regexp.py	Tue Mar 01 21:09:36 2022 +0100
+++ b/pytst/readers/parser/d1_parse_tree/test_1_regexp.py	Sun Mar 06 18:45:19 2022 +0100
@@ -18,6 +18,9 @@
     if pattern:
         validate_pattern(parse_tree, pattern=pattern)
 
+    return parse_tree
+
+
 def validate_pattern(parse_tree, pattern):
         for e,T in zip(parse_tree,pattern):
             if T is not None: assert isinstance(e.rule, T), f"{type(e.rule).__name__}  doesn't match {T.__name__}"
@@ -36,5 +39,24 @@
 def test_grammar_re_no_slash(): verify_regex(r"/((\\/)|[^\/])*/")
 def test_grammar_auto_re_no_slash(): verify_regex("/" + grammar.re_no_slash().to_match +"/") # Same as above (unless grammar changes
 
+def test_any2EOL_1():
+    pt = verify_regex("/.*\n/") # NOT raw : \n === newline
+    re = pt[1]
+    assert re == ".*\n"
+
+def test_any2EOL_2():
+    pt = verify_regex("""/.*
+/""") #Explicit newline (in not-raw string
+    re = pt[1]
+    assert re == ".*\n"
+
+def test_any2EOL_3():
+    pt = verify_regex(r"""/.*
+/""") #Explicit newline (in RAW string
+    re = pt[1]
+    assert re == ".*\n"
 
 
+
+
+

diff -r 76f7a4c5a47d -r 54ef2ab8a19e pytst/readers/parser/d1_parse_tree/test_9_file.py
--- a/pytst/readers/parser/d1_parse_tree/test_9_file.py	Tue Mar 01 21:09:36 2022 +0100
+++ b/pytst/readers/parser/d1_parse_tree/test_9_file.py	Sun Mar 06 18:45:19 2022 +0100
@@ -1,4 +1,5 @@
 import pytest
+import logging; logger = logging.getLogger(__name__)
 from pathlib import Path
 import os
 
@@ -9,6 +10,7 @@
     path_to_current_test = Path(os.path.realpath(__file__))
     path_to_current_dir = path_to_current_test.parent
     with (path_to_current_dir / dir / filename).open() as f:
+        logger.debug(f'Reading file: >>{f.name}<<')
         txt = f.read()
 
     parser = arpeggio.ParserPython(grammar.peg_grammar, grammar.comment, debug=False)

diff -r 76f7a4c5a47d -r 54ef2ab8a19e pytst/readers/parser/d2_ast/test_1_term.py
--- a/pytst/readers/parser/d2_ast/test_1_term.py	Tue Mar 01 21:09:36 2022 +0100
+++ b/pytst/readers/parser/d2_ast/test_1_term.py	Sun Mar 06 18:45:19 2022 +0100
@@ -1,4 +1,5 @@
 import pytest
+import logging; logger = logging.getLogger(__name__)
 
 from castle.readers.parser import grammar
 from castle.ast import peg
@@ -57,3 +58,12 @@
     assert isinstance(ast, peg.Expression),	"A (str)term is also an Expression"
     assert len(ast) == 1,			"with a lengt of 1 -- note: use: ``len(sequence)`` not ``len(sequence._children)``!!"
     assert ast[0].value == txt[1:-1], 	        "It's correct value should be without quotes"
+
+
+def test_any2EOL():
+    txt = "/.*\n/" # NOT raw : \n === newline
+    ast = parse(txt, grammar.term)
+    logger.debug(f"any2EOL_1:: {ast}")
+
+    assert isinstance(ast, peg.RegExpTerm)
+    assert ast.value == ".*\n"

diff -r 76f7a4c5a47d -r 54ef2ab8a19e pytst/readers/parser/general/test_file.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pytst/readers/parser/general/test_file.py	Sun Mar 06 18:45:19 2022 +0100
@@ -0,0 +1,24 @@
+import pytest
+import logging; logger = logging.getLogger(__name__)
+
+from castle.readers.parser import PEGReader
+from castle.readers.parser import grammar, visitor
+
+
+def test_file():
+    reader = PEGReader(read_dirs=('../../../demos/ThinOnion',
+                                  '../../.././demos/ThinOnion/grammar',
+                                  '../../.././pytst/readers/parser'),
+                       language_def=grammar.peg_grammar,
+                       comment_def=grammar.comment,
+                       visitor=visitor.PegVisitor())
+    ast = reader.parse('grammar.peg')
+    # Remember: ast is a peg.Grammar!!
+
+    #Manuel count ...
+    no_rules, no_settings = 20,19
+    assert len(ast.parse_rules) == no_rules, f"The number of (real) rules should be {no_rules} -- unless the file is changed"
+    assert len(ast.settings) == no_settings, f"The number of settings ('=') should be {no_settings} -- unless the file is changed"
+
+    # XXX See grammar.peg :: comment doesn't works
+

diff -r 76f7a4c5a47d -r 54ef2ab8a19e pytst/readers/parser/grammar.peg
--- a/pytst/readers/parser/grammar.peg	Tue Mar 01 21:09:36 2022 +0100
+++ b/pytst/readers/parser/grammar.peg	Sun Mar 06 18:45:19 2022 +0100
@@ -1,42 +1,60 @@
-//This is the PEG version og grammar(2).py
-
-peg_grammar	<-  rules ;
-rules		<-  rule+ ;
-rule		<-  rule_name '<-' expression ';' ;
+peg_grammar	<-  rules EOF ;
+rules		<-  (parse_rule | setting)+ ;
+parse_rule	<-  rule_name '<-' expression ';' ;
 
 expression	<-  sequence op_alternative ;
 sequence        <-  single_expr+ ;
 single_expr	<-  ( rule_crossref | term | group | predicate )  op_quantity ;
 
 op_alternative  <-  ( '|' expression )? ;
-op_quantity     <-  ( '?' | '*' | '+' | '#' )? ;
+op_quantity     <-  ( '?' | '*' | '+' | '/#' )? ;
 
 term		<-  str_term | regex_term ;
 group		<-  '(' expression ')' ;
 predicate	<-   ( '&' | '!' ) single_expr ;
 
-str_term	<- "'"    str_no_s1  "'"
-		|  '"'    str_no_d1  '"'
-		|  "'''"  str_no_s3  "'''"
-		|  '"""'  str_no_d3  '"""'
+str_term	<- S3  str_no_s3  S3
+		|  D3  str_no_d3  D3
+		|  S1  str_no_s1  S1
+		|  D1  str_no_d1  D1
 		;
-regex_term	<- '/'     str_no_slash  '/'
-		|  "r'"    str_no_s1  "'"
-		|  'r"'    str_no_d1  '"'
-		|  "r'''"  str_no_s3  "'''"
-		|  'r"""'  str_no_d3  '"""'
-		|  "R'"    str_no_s1  "'"
-		|  'R"'    str_no_d1  '"'
-		|  "R'''"  str_no_s3  "'''"
-		|  'R"""'  str_no_d3  '"""'
+regex_term	<- RE    re_no_slash  RE
+                |  REs3  str_no_s3     S3
+                |  REd3  str_no_d3     D3
+                |  REs1  str_no_s1     S1
+                |  REd1  str_no_d1     D1
 		;
 
-rule_crossref	<- ID ;
-rule_name	<- ID ;
-ID		<- /[A-Za-z_][A-Za-z0-9_]*/ ;
+rule_name	= ID ;
+rule_crossref	= ID ;
+ID		= /[A-Za-z_][A-Za-z0-9_]*/ ;
 
-str_no_slash	<- /((\\/)|[^\/])*/ ;
-str_no_XXX	<- XXX
-		|  ETC
-		;
+re_no_slash 	=  /((\\/)|[^\/])*/          ;
+str_no_s1	= /((\\')|[^'\n])*/          ;
+str_no_d1	= /((\\")|[^"\n])*/          ;
+str_no_s3	= /([^']|('[^'])|(''[^']))*/ ;
+str_no_d3	= /([^"]|("[^"])|(""[^"]))*/ ;
 
+
+setting		<- setting_name  '=' value ';' ;
+setting_name	=  ID ;
+value           <- str_term | regex_term | number | setting_xref ;
+number		<- complex_lit | float_lit | int_lit ;
+setting_xref	=  ID ;
+complex_lit	<- /[+-]?([0-9](\.[0-9]*)?)[+-][iIjJ]([0-9](\.[0-9]*)?)/ ;
+float_lit	<- /[+-]?[0-9]\.[0-9]+/;
+int_lit		<- /[+-]?[1-9][0-9]*/ ;
+
+S1 = "'" ;
+D1 = '"' ;
+S3 = "'''" ;
+D3 = '"""' ;
+RE   = '/' ;
+REs1 = /[rR]'"/ ;
+REd1 = /[rR]"/ ;
+REs3 = /[rR]'''/ ;
+REd3 = /[rR]"""/ ;
+
+comment		<- comment1 | comment2 ;
+#comment1	<- '#'   /.*\n/        ; // XXX:: Does Not work
+#comment2	<- '//'  /.*\n/        ; # XXX:: Does Not work