add do-while and f-string dynamic addition proofs of concepts

Makefile

@@ -5,7 +5,7 @@ ifeq ($(UNAME_S),Darwin)
 endif
 
 GFLAGS += -Wall -Wno-unused -g
-OFLAGS += $(GFLAGS) -O3 -DNDEBUG -DTYPECODES=1 -DDELOPT=1
+OFLAGS += $(GFLAGS) -O3 -DNDEBUG -DTYPECODES=1 -DDELOPT=1 -Wno-array-bounds
 PFLAGS += $(OFLAGS) -pg
 CFLAGS += -D_GNU_SOURCE -I/opt/local/include
 LDLIBS += -L/opt/local/lib
@@ -15,7 +15,7 @@ MAIN = minproto
 
 all : $(MAIN)
 
-% : %.c
+% : %.c %.grammar
 	$(CC) $(GFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ $< $(LDLIBS)
 
 %-opt : %.c
@@ -27,6 +27,9 @@ all : $(MAIN)
 %.c : %.leg
 	leg -o $@ $<
 
+%.grammar : %.leg
+	( sed "/%{/,/%}/d" $< | sed "/%%/,\$$d" ) > $@
+
 tests : .FORCE
 	$(MAKE) clean all GFLAGS="-Wno-unused -g -DTYPECODES=1 -DELOPT=1"
 	-./$(MAIN) test.txt > test.out 2>&1

bench-sieve.txt

@@ -3,7 +3,7 @@
 // 178589010 nodes evaluated in 2.126711 seconds = 83974273 nodes/sec
 
 numbers = 8192; // 1028 primes
-repeats = 10; // 200;
+repeats = 200;
 
 count = 0;
 

dowhile.meta

@@ -0,0 +1,56 @@
+Do = Object.subtype(#Do);
+
+Do.new(body, cond) {
+    self = super.new();
+    self.body = body;
+    self.cond = cond;
+    self;
+}
+
+Do.__eval__(env) {
+    local result = self.body.__eval__(env);
+    while (self.cond.__eval__(env)) {
+        result = self.body.__eval__(env);
+    }
+    result;
+}
+
+Do.__codeon__(str) {
+    str.push("do ");
+    self.body.__codeon__(str);
+    str.push(" while ( ");
+    self.cond.__codeon__(str);
+    str.push(" )");
+}
+
+// parse the grammar expression
+// put it at the beginning of stmt (which is an alternation)
+__namespaces__.metaLanguage.stmt.prepend("\"whatever i want\" - b:stmt WHILE LPAREN c:expr RPAREN EOS { Do.new(b, c); }");
+
+// regenerate the expression to be executed
+metaLanguage.addRule(#stmt, __namespaces__.metaLanguage.stmt);
+
+{
+
+local i = 1;
+
+    {
+
+        whatever i want {
+            i *= 2;
+            print(i, "\n");
+        } while (i < 1000);
+
+        whatever i want {
+            i *= 2;
+            print(i, "\n");
+        } while (i < 1000);
+
+        whatever i want {
+            i *= 2;
+            print(i, "\n");
+        } while (i < 1000);
+
+    }
+
+}

fstring.meta

@@ -0,0 +1,80 @@
+FormatString = Object.subtype(#FormatString);
+
+FormatString.new(elements) {
+    self = super.new();
+    self.elements = elements;
+    self;
+}
+
+FormatString.__eval__(env) {
+    local result = "";
+    for(elem in self.elements) {
+        local elemval = elem.__eval__(env);
+        if (elemval.__name__ == #String) {
+            result.push(elemval);
+        } else {
+            elemval.__codeon__(result);
+        }
+    }
+    result;
+}
+
+FormatString.__codeon__(str) {
+    str.push("f\"");
+    for(elem in self.elements) {
+        if (elem.__name__ == #String) {
+            str.push(elem.escaped());
+        } else {
+            str.push("$");
+            elem.__codeon__(str);
+        }
+    }
+    str.push("\"");
+}
+
+__namespaces__.metaLanguage.primary.prepend("fstring");
+
+nonSpaceEatingId = parseDefinition(
+"nonSpaceEatingId = < LETTER ALNUM* >           { intern(yytext) }
+");
+
+nonSpaceEatingBlock = parseDefinition(
+"nonSpaceEatingBlock = LBRACE b:mkobj
+                     (  e:stmt                  { b.push(e) }
+                     )* \"}\"                   { b }
+");
+
+fStringChar = parseDefinition(
+"fStringChar = < (!\"\\\"\" !\"$\" char )+ > { yytext }
+             | \"\\\\$\"                     { \"$\" }
+");
+
+fStringRule = parseDefinition(
+"fstring = \"f\\\"\" elements:mkobj
+        ( \"$\" i:nonSpaceEatingId              { elements.push(GetVar.new(name: i)) }
+        | \"$\" b:nonSpaceEatingBlock           { elements.push(Block.new(body: b)) }
+        | c:fStringChar                         { elements.push(c) }
+        )* \"\\\"\"                             { FormatString.new(elements) }
+");
+
+// regenerate the expression to be executed
+metaLanguage.addRule(nonSpaceEatingBlock.name, nonSpaceEatingBlock.expression);
+metaLanguage.addRule(nonSpaceEatingId.name, nonSpaceEatingId.expression);
+metaLanguage.addRule(fStringChar.name, fStringChar.expression);
+metaLanguage.addRule(fStringRule.name, fStringRule.expression);
+metaLanguage.addRule(#primary, __namespaces__.metaLanguage.primary);
+
+formatInt(integer) {
+    if (integer > 1000) {
+        return f"${formatInt((integer - integer % 1000) / 1000)},${integer % 1000}";
+    }
+    return f"$integer";
+}
+a = 69;
+print(f"Reimu has $a power and ${6*7} point items and \$${ formatInt(1 << 24) } money.\n");
+n = 100;
+print(f"Sum of integers from 1 to $n : ${local sum = 0; for (i in n + 1) sum += i}\n");
+print(f"${formatInt.function}\n");
+//{
+//    print(f"${__env__()}\n");
+//}

minproto.grammar

@@ -0,0 +1,316 @@
+# minproto.leg -- minimal prototype langauge for semantic experiments
+#
+# last edited: 2024-07-05 17:16:16 by piumarta on zora-1034.local
+
+
+start   = - ( s:stmt                            { global yysval = s }
+            | !.                                { global yysval = 0 }
+            | < (!EOL .)* >                     { syntaxError(yytext) }
+            )
+
+stmt    = WHILE LPAREN c:expr RPAREN s:stmt     { $$ = newWhile(c, s) }
+        | IF LPAREN c:expr RPAREN s:stmt
+          ( ELSE t:stmt                         { $$ = newIf(c, s, t  ) }
+          |                                     { $$ = newIf(c, s, nil) }
+          )
+        | CONT EOS                              { $$ = newContinue() }
+        | BREAK e:expr EOS                      { $$ = newBreak(e) }
+        | BREAK EOS                             { $$ = newBreak(nil) }
+        | RETURN e:expr EOS                     { $$ = newReturn(e) }
+        | RETURN EOS                            { $$ = newReturn(nil) }
+        | FOR LPAREN i:id IN e:expr RPAREN
+              s:stmt                            { $$ = newForIn(i, e, s) }
+        | FOR LPAREN i:id FROM a:expr
+                 TO  b:expr RPAREN s:stmt       { $$ = newForFromTo(i, a, b, s) }
+        | FOR LPAREN i:expr SEMI c:expr SEMI
+                     u:expr RPAREN s:stmt       { $$ = newFor(i, c, u, s) }
+        | TRY t:stmt
+          ( CATCH LPAREN i:id RPAREN c:stmt     { $$ = newTryCatch(t, i, c) }
+          | ENSURE e:stmt                       { $$ = newTryEnsure(t, e) }
+          )
+        | RAISE e:expr EOS                      { $$ = newRaise(e) }
+        | LOCAL       i:id p:params b:block     { $$ = newSetLocal (i, newLambda(p, b, nil, i)) }
+        | GLOBAL      i:id p:params b:block     { $$ = newSetGlobal(i, newLambda(p, b, nil, i)) }
+        |             i:id p:params b:block     { $$ = newSetVar   (i, newLambda(p, b, nil, i)) }
+        | v:proto DOT i:id p:params b:block     { $$ = newSetProp(v, i, newLambda(p, b, v, i)) }
+        | b:block                               { $$ = newBlock(b) }
+        | e:expr EOS                            { $$ = e }
+
+proto   = v:var ( DOT j:id !LPAREN              { v = newGetProp(v, j) }
+                )*                              { $$ = v }
+
+EOS     = SEMI+ | &RBRACE | &ELSE | &CATCH
+
+expr    = LOCAL  i:id ASSIGN e:expr                    { $$ = newSetLocal (i, e) }
+        | GLOBAL i:id ASSIGN e:expr                    { $$ = newSetGlobal(i, e) }
+        |        i:id ASSIGN e:expr                    { $$ = newSetVar   (i, e) }
+        | l:logor ( ASSIGN   r:expr             { l  = assign(l, r) }
+                  | PLUSEQ   r:expr             { l  = newBinop(opPreAdd, lvalue(l), r) }
+                  | MINUSEQ  r:expr             { l  = newBinop(opPreSub, lvalue(l), r) }
+                  | STAREQ   r:expr             { l  = newBinop(opPreMul, lvalue(l), r) }
+                  | SLASHEQ  r:expr             { l  = newBinop(opPreDiv, lvalue(l), r) }
+                  | PCENTEQ  r:expr             { l  = newBinop(opPreMod, lvalue(l), r) }
+                  | SHLEQ    r:expr             { l  = newBinop(opPreShl, lvalue(l), r) }
+                  | SHREQ    r:expr             { l  = newBinop(opPreShr, lvalue(l), r) }
+                  | ANDEQ    r:expr             { l  = newBinop(opPreAnd, lvalue(l), r) }
+                  | XOREQ    r:expr             { l  = newBinop(opPreXor, lvalue(l), r) }
+                  | OREQ     r:expr             { l  = newBinop(opPreOr,  lvalue(l), r) }
+                  )?                            { $$ = l }
+
+logor   = l:logand ( BARBAR r:logand            { l = newBinop(opLogOr,  l, r) }
+                   )*                           { $$ = l }
+
+logand  = l:bitor ( ANDAND r:bitor              { l = newBinop(opLogAnd, l, r) }
+                  )*                            { $$ = l }
+
+bitor   = l:bitxor ( OR r:bitxor                { l = newBinop(opBitOr,  l, r) }
+                   )*                           { $$ = l }
+
+bitxor  = l:bitand ( XOR r:bitand               { l = newBinop(opBitXor, l, r) }
+                   )*                           { $$ = l }
+
+bitand  = l:eq ( AND r:eq                       { l = newBinop(opBitAnd, l, r) }
+               )*                               { $$ = l }
+
+eq      = l:ineq ( EQ    r:ineq                 { l = newBinop(opEq,    l, r) }
+                 | NOTEQ r:ineq                 { l = newBinop(opNotEq, l, r) }
+                 )*                             { $$ = l }
+
+ineq    = l:shift ( LESS   r:shift              { l = newBinop(opLess,   l, r) }
+                  | LESSEQ r:shift              { l = newBinop(opLessEq, l, r) }
+                  | GRTREQ r:shift              { l = newBinop(opGrtrEq, l, r) }
+                  | GRTR   r:shift              { l = newBinop(opGrtr,   l, r) }
+                  )*                            { $$ = l }
+
+shift   = l:sum ( SHL r:sum                     { l = newBinop(opShl, l, r) }
+                | SHR r:sum                     { l = newBinop(opShr, l, r) }
+                )*                              { $$ = l }
+
+sum     = l:prod ( PLUS  r:prod                 { l = newBinop(opAdd, l, r) }
+                 | MINUS r:prod                 { l = newBinop(opSub, l, r) }
+                 )*                             { $$ = l }
+
+prod    = l:prefix ( STAR r:prefix              { l = newBinop(opMul, l, r) }
+                  | SLASH r:prefix              { l = newBinop(opDiv, l, r) }
+                  | PCENT r:prefix              { l = newBinop(opMod, l, r) }
+                  ) *                           { $$ = l }
+
+prefix  = PPLUS  p:prefix                       { $$ = newBinop(opPreAdd, lvalue(p), newInteger(1)) }
+        | MMINUS p:prefix                       { $$ = newBinop(opPreSub, lvalue(p), newInteger(1)) }
+        | PLING  p:prefix                       { $$ = newUnyop(opNot, p) }
+        | MINUS  p:prefix                       { $$ = newUnyop(opNeg, p) }
+        | TILDE  p:prefix                       { $$ = newUnyop(opCom, p) }
+        | BQUOTE s:stmt                         { $$ = newUnyop(opQuasiquote, s) }
+        | COMMAT e:expr                         { $$ = newUnyop(opUnquote, e) }
+        | postfix
+
+postfix = SUPER DOT i:id a:args                 { $$ = newSuper(i, a) }
+        | p:primary
+          ( LBRAK
+            (           COLON (         RBRAK   { p = newGetSlice(p, nil, nil) }
+                              | e:xexpr RBRAK   { p = newGetSlice(p, nil, e) }
+                              )
+            | s:xexpr ( COLON (         RBRAK   { p = newGetSlice(p, s, nil) }
+                              | e:xexpr RBRAK   { p = newGetSlice(p, s, e) }
+                              )
+                      |                 RBRAK   { p = newGetArray(p, s) }
+                      )
+            )
+          | DOT   i:id ( a:args !LBRACE         { p = newInvoke(p, i, a) }
+                       |                        { p = newGetProp(p, i) }
+                       )
+          |              a:args !LBRACE         { p = newApply(p, a) }
+          )*
+          ( PPLUS                               { p  = newBinop(opPostAdd, lvalue(p), newInteger( 1)) }
+          | MMINUS                              { p  = newBinop(opPostAdd, lvalue(p), newInteger(-1)) }
+          )?                                    { $$ = p }
+
+args    = LPAREN a:mkobj
+          ( RPAREN
+          | ( k:id COLON e:xexpr                { Object_put(a, k, e) }
+            | e:xexpr                           { Object_push(a, e) }
+            )
+            ( COMMA ( k:id COLON e:xexpr        { Object_put(a, k, e) }
+                    | e:xexpr                   { Object_push(a, e) }
+                    ) )* RPAREN )               { $$ = a }
+
+params  = LPAREN p:mkobj
+          ( RPAREN
+          | i:id ( COLON e:expr                 { Object_put(p, i, e) }
+                 |                              { Object_push(p, i) }
+                 )
+            ( COMMA i:id ( COLON e:expr         { Object_put(p, i, e) }
+                         |                      { Object_push(p, i) }
+                         )
+            )* RPAREN )                         { $$ = p }
+
+mkobj   =                                       { $$ = (global new)(pObject) }
+
+primary = nil | number | string | symbol | var | lambda | subexpr | literal # | regex
+
+lambda  = p:params b:block                      { $$ = newLambda(p, b, nil, nil) }
+
+subexpr = LPAREN e:expr  RPAREN                 { $$ = e }
+        |        b:block                        { $$ = newBlock(b) }
+
+literal = LBRAK o:mkobj
+          ( RBRAK
+          | ( ( i:id COLON e:expr               { Object_put(o, i, e) }
+              | e:expr                          { Object_push(o, e) }
+              ) ( COMMA ( i:id COLON e:expr     { Object_put(o, i, e) }
+                        | e:expr                { Object_push(o, e) }
+                        ) )* )? RBRAK )         { $$ = newLiteral(o) }
+
+block   = LBRACE b:mkobj
+            (  e:stmt                           { Object_push(b, e) }
+            )* ( RBRACE                         { $$ = b }
+               | error                         @{ expected("statement or \x7D", yytext) }
+               )
+
+nil     = NIL                                   { $$ = nil }
+
+number  = "-" n:unsign                          { $$ = neg(n) }
+        | "+" n:number                          { $$ = n }
+        |     n:unsign                          { $$ = n }
+
+unsign  = < DIGIT* '.' DIGIT+ EXP? > -          { $$ = newFloat(strtod(yytext, 0)) }
+        | "0" [bB] < BIGIT+ > -                 { $$ = newInteger(strtol(yytext, 0,  2)) }
+        | "0" [xX] < HIGIT+ > -                 { $$ = newInteger(strtol(yytext, 0, 16)) }
+        | "0" < OIGIT* > -                      { $$ = newInteger(strtol(yytext, 0,  8)) }
+        | < DIGIT+ > -                          { $$ = newInteger(strtol(yytext, 0, 10)) }
+        | "'" < char > "'" -                    { $$ = newInteger(_get(newStringUnescaped(yytext), String,value)[0]) }
+
+string  = '"' < ( !'"' char )* > '"' -          { $$ = newStringUnescaped(yytext) }
+
+char           = '\\' [abefnrtv'"\[\]\\]
+               | '\\' [0-3][0-7][0-7]
+               | '\\' [xX] HIGIT*
+               | '\\' [0-7][0-7]?
+               | !'\\' .
+# char    = "\\" ( ["'\\abfnrtv]
+#                | [xX] HIGIT*
+#                | [0-7][0-7]?[0-7]?
+#                )
+#         | .
+
+symbol  = HASH i:id                             { $$ = i }
+
+var     = LOCAL  i:id                           { $$ = newGetLocal (i) }
+        | GLOBAL i:id                           { $$ = newGetGlobal(i) }
+        |        i:id                           { $$ = newGetVar   (i) }
+
+id      = < LETTER ALNUM* > -                   { $$ = intern(yytext) }
+
+# regex   = SLASH a:alts SLASH                    { $$ = a }
+
+# alts    = s:seq ( OR t:seq                      { s = Alt_append(t) }
+#                 )*                              { $$ = s }
+
+# seq     = p:pre ( q:pre                         { s = Seq_append(t) }
+#                 )*                              { $$ = s }
+
+# elt     = action | pre
+
+# action  = b:block                               { $$ = newAction(b) }
+
+# pre     = PLING p:pre                           { $$ = newNot(p) }
+#         | AND   p:pre                           { $$ = newAnd(p) }
+#         | post
+
+# post    = a:atom ( STAR                         { a = newMany(a) }
+#                  | PLUS                         { a = newMore(a) }
+#                  | QUERY                        { a = newMore(a) }
+#                  )?                             { $$ = a }
+
+# atom    = DOT                                   { $$ = newDot() }
+#         | "[" ( !"]" "\\"? . )* "]" -           { $$ = newClass(yytext) }
+#         | '"' xxxxxx
+
+# class   = LBRAK 
+
+BIGIT   = [0-1]
+OIGIT   = [0-7]
+DIGIT   = [0-9]
+HIGIT   = [0-9A-Fa-f]
+LETTER  = [A-Za-z_$?]
+ALNUM   = LETTER | DIGIT
+SIGN    = [-+]
+EXP     = [eE] SIGN DIGIT+
+
+-       = SPACE*
+
+SPACE   = [ \t] | EOL | SLC | MLC
+EOL     = [\n\r]                                { ++lineno }
+SLC     = "//" (!EOL .)*
+MLC     = "/*" ( MLC | !"*/" (EOL | .))* "*/" -
+
+NIL     = "nil"      !ALNUM  -
+WHILE   = "while"    !ALNUM  -
+IF      = "if"       !ALNUM  -
+ELSE    = "else"     !ALNUM  -
+FOR     = "for"      !ALNUM  -
+IN      = "in"       !ALNUM  -
+FROM    = "from"     !ALNUM  -
+TO      = "to"       !ALNUM  -
+CONT    = "continue" !ALNUM  -
+BREAK   = "break"    !ALNUM  -
+RETURN  = "return"   !ALNUM  -
+TRY     = "try"      !ALNUM  -
+CATCH   = "catch"    !ALNUM  -
+ENSURE  = "ensure"   !ALNUM  -
+RAISE   = "raise"    !ALNUM  -
+GLOBAL  = "global"   !ALNUM  -
+LOCAL   = "local"    !ALNUM  -
+SUPER   = "super"    !ALNUM  -
+
+BQUOTE  = "`"           -
+COMMAT  = "@"           -
+HASH    = "#"           -
+SEMI    = ";"           -
+ASSIGN  = "="  ![=]     -
+COMMA   = ","           -
+COLON   = ":"  ![:]     -
+LPAREN  = "("           -
+RPAREN  = ")"           -
+LBRAK   = "["           -
+RBRAK   = "]"           -
+LBRACE  = "{"           -
+RBRACE  = "}"           -
+BARBAR  = "||" ![=]     -
+ANDAND  = "&&" ![=]     -
+OR      = "|"  ![|=]    -
+OREQ    = "|="          -
+XOR     = "^"  ![=]     -
+XOREQ   = "^="          -
+AND     = "&"  ![&=]    -
+ANDEQ   = "&="          -
+EQ      = "=="          -
+NOTEQ   = "!="          -
+LESS    = "<"  ![<=]    -
+LESSEQ  = "<="          -
+GRTREQ  = ">="          -
+GRTR    = ">"  ![=]     -
+SHL     = "<<" ![=]     -
+SHLEQ   = "<<="         -
+SHR     = ">>" ![=]     -
+SHREQ   = ">>="         -
+PLUS    = "+"  ![+=]    -
+PLUSEQ  = "+="          -
+PPLUS   = "++"          -
+MINUS   = "-"  ![-=]    -
+MINUSEQ = "-="          -
+MMINUS  = "--"          -
+STAR    = "*"  ![=]     -
+STAREQ  = "*="          -
+SLASH   = "/"  ![/=]    -
+SLASHEQ = "/="          -
+PCENT   = "%"  ![=]     -
+PCENTEQ = "%="          -
+DOT     = "."  ![.]     -
+PLING   = "!"  ![=]     -
+TILDE   = "~"           -
+
+error   = - < (!EOL .)* >
+
+xexpr   = expr | error @{ expected("expression", yytext) }
+

minproto.leg

@@ -1,9 +1,9 @@
 # minproto.leg -- minimal prototype langauge for semantic experiments
 #
-# last edited: 2024-06-20 13:22:50 by piumarta on m1mbp
+# last edited: 2024-07-05 17:16:16 by piumarta on zora-1034.local
 
 %{
-    ;
+;
 //#define YY_DEBUG 1
 
 #ifndef GC
@@ -121,9 +121,9 @@ oop printOn(oop buf, oop obj, int indent);
 #endif
 
 #if PRIMCLOSURE
-#define doProtos(_) _(Object) _(RefLocal) _(GetLocal) _(SetLocal) _(RefGlobal) _(GetGlobal) _(SetGlobal) _(RefVar) _(GetVar) _(SetVar) _(RefProp) _(GetProp) _(SetProp) _(RefArray) _(GetArray) _(SetArray) _(GetSlice) _(Call) _(Invoke) _(Super) _(Continue) _(Break) _(Return) _(TryCatch) _(TryEnsure)_(Raise)_(Binop) _(Unyop) _(If) _(While) _(Block) _(For) _(ForIn) _(ForFromTo) _(Literal)
+#define doProtos(_) _(Object) _(RefLocal) _(GetLocal) _(SetLocal) _(RefGlobal) _(GetGlobal) _(SetGlobal) _(RefVar) _(GetVar) _(SetVar) _(RefProp) _(GetProp) _(SetProp) _(RefArray) _(GetArray) _(SetArray) _(GetSlice) _(Call) _(Invoke) _(Super) _(Continue) _(Break) _(Return) _(TryCatch) _(TryEnsure)_(Raise)_(Binop) _(Unyop) _(If) _(While) _(Block) _(For) _(ForIn) _(ForFromTo) _(Literal) _(Stream)
 #else
-#define doProtos(_) _(Object) _(RefLocal) _(GetLocal) _(SetLocal) _(RefGlobal) _(GetGlobal) _(SetGlobal) _(RefVar) _(GetVar) _(SetVar) _(RefProp) _(GetProp) _(SetProp) _(RefArray) _(GetArray) _(SetArray) _(GetSlice) _(Call) _(Invoke) _(Super) _(Continue) _(Break) _(Return) _(TryCatch) _(TryEnsure) _(Raise) _(Binop) _(Unyop) _(If) _(While) _(Block) _(For) _(ForIn) _(ForFromTo) _(Literal) _(Lambda) _(Closure)
+#define doProtos(_) _(Object) _(RefLocal) _(GetLocal) _(SetLocal) _(RefGlobal) _(GetGlobal) _(SetGlobal) _(RefVar) _(GetVar) _(SetVar) _(RefProp) _(GetProp) _(SetProp) _(RefArray) _(GetArray) _(SetArray) _(GetSlice) _(Call) _(Invoke) _(Super) _(Continue) _(Break) _(Return) _(TryCatch) _(TryEnsure) _(Raise) _(Binop) _(Unyop) _(If) _(While) _(Block) _(For) _(ForIn) _(ForFromTo) _(Literal) _(Lambda) _(Closure) _(Stream)
 #endif
 
 #define  declareProto(NAME)     oop p##NAME = 0;
@@ -147,7 +147,7 @@ doTypes(makeProto);
 doProperties(declareProp);
 #undef declareProp
 
-#define doSymbols(_) _(t) _(name) _(expr) _(function) _(arguments) _(object) _(index) _(key) _(value) _(self) _(method) _(parameters) _(body) _(lambda) _(environment) _(operation) _(full) _(condition) _(consequent) _(alternate) _(expression) _(identifier) _(initialise) _(update) _(first) _(last) _(fixed) _(keyvals) _(__namespaces__) _(O) _(d) _(p) _(v) _(statement) _(handler) _(kind) _(message) _(operand1) _(operand2) _(profile) _(parent) _(count) _(stamp) _(time) _(start) _(stop) _($$) _(yytext) _(yyleng)
+#define doSymbols(_) _(t) _(name) _(expr) _(function) _(arguments) _(object) _(index) _(key) _(value) _(self) _(method) _(parameters) _(body) _(lambda) _(environment) _(operation) _(full) _(condition) _(consequent) _(alternate) _(expression) _(identifier) _(initialise) _(update) _(first) _(last) _(fixed) _(keyvals) _(__namespaces__) _(O) _(d) _(p) _(v) _(statement) _(handler) _(kind) _(message) _(operand1) _(operand2) _(profile) _(parent) _(count) _(stamp) _(time) _(start) _(stop) _($$) _(yytext) _(yyleng) _(env) _(content) _(position) _(limit) _(lastBegin)
 
 #define declareSym(NAME)        oop sym_##NAME = 0;
 doSymbols(declareSym);
@@ -387,10 +387,11 @@ oop String_append(oop str, int c)
 {
     int  length = get(str, String,length);
     char *value = get(str, String,value);
-    value = xrealloc(value, length + 1);
-    set(str, String,value,  value);
+    char *copy  = xmalloc(length + 1);
+    memcpy(copy, value, length);
+    set(str, String,value,  copy);
     set(str, String,length, length+1);
-    value[length] = c;
+    copy[length] = c;
     return str;
 }
 
@@ -399,9 +400,10 @@ oop String_appendAllLen(oop str, char *s, int len)
     if (len < 1) return str;
     int  length = get(str, String,length);
     char *value = get(str, String,value);
-    value = xrealloc(value, length + len);
-    memcpy(value + length, s, len);
-    set(str, String,value,  value);
+    char *copy  = xmalloc(length + len);
+    memcpy(copy, value, length);
+    memcpy(copy + length, s, len);
+    set(str, String,value,  copy);
     set(str, String,length, length+len);
     return str;
 }
@@ -422,7 +424,9 @@ oop String_format(oop str, char *fmt, ...)
     int  length = get(str, String,length);
     char *value = get(str, String,value);
     for (;;) {
-        value = xrealloc(value, length + cap);
+        char *orig = value;
+        value = xmalloc(length + cap);
+        memcpy(value, orig, length);
         va_list ap;
         va_start(ap, fmt);
         len = vsnprintf(value + length, cap, fmt, ap);
@@ -497,6 +501,8 @@ oop newStringUnescaped(char *string)
                 case 'r' : c = '\r';  break;
                 case 't' : c = '\t';  break;
                 case 'v' : c = '\v';  break;
+                case '[' : c = '[' ;  break;
+                case ']' : c = ']' ;  break;
                 case 'X' :
                 case 'x' :      c = readCharValue(&string, 16, -1);                          break;
                 case '0'...'7': --string;  c = readCharValue(&string, 8, 3);                 break;
@@ -767,7 +773,7 @@ char *storeString(oop obj, int indent);
 
 oop *Object_ref(oop obj, oop key)
 {
-    oop o;
+    oop o = nil;
     switch (getType(obj)) {
         case Undefined: o = pUndefined;  break;
         case Integer:   o = pInteger;    break;
@@ -810,7 +816,7 @@ oop *Object_ref(oop obj, oop key)
 
 oop Object_getOwner(oop obj, oop key, oop *ownerp)
 {
-    oop o;
+    oop o = nil;
     switch (getType(obj)) {
         case Undefined: o = pUndefined;  break;
         case Integer:   o = pInteger;    break;
@@ -853,7 +859,7 @@ oop Object_getOwner(oop obj, oop key, oop *ownerp)
 
 oop Object_get(oop obj, oop key)
 {
-    oop o;
+    oop o = nil;
     switch (getType(obj)) {
         case Undefined: o = pUndefined;  break;
         case Integer:   o = pInteger;    break;
@@ -3336,6 +3342,26 @@ void Literal_codeOn(oop exp, oop str, oop env)
 # endif
 }
 
+oop newStream(oop content)
+{
+    oop o = new(pStream);
+    Object_put(o, sym_content  , content);
+    Object_put(o, sym_position , newInteger(0));
+    Object_put(o, sym_limit    , newInteger(_get(content, String, length)));
+    Object_put(o, sym_lastBegin, newInteger(0));
+    return o;
+}
+
+oop Stream_eval(oop exp, oop env)
+{
+    return exp;
+}
+
+void Stream_codeOn(oop exp, oop str, oop env)
+{
+    Object_codeOn(exp, str, env);
+}
+
 oop lvalue(oop rval)
 {
     if (!is(Object,rval)) valueError("=", "non-assignable value", rval);
@@ -3368,10 +3394,12 @@ void expected(char *what, char *where)
     fatal("syntax error: %s expected near: %s", what, where);
 }
 
+#define global
+
 %}
 
-start   = - ( s:stmt                            { yysval = s }
-            | !.                                { yysval = 0 }
+start   = - ( s:stmt                            { global yysval = s }
+            | !.                                { global yysval = 0 }
             | < (!EOL .)* >                     { syntaxError(yytext) }
             )
 
@@ -3457,10 +3485,10 @@ sum     = l:prod ( PLUS  r:prod                 { l = newBinop(opAdd, l, r) }
                  | MINUS r:prod                 { l = newBinop(opSub, l, r) }
                  )*                             { $$ = l }
 
-prod    = l:prefix ( STAR  r:prefix             { l = newBinop(opMul, l, r) }
-                   | SLASH r:prefix             { l = newBinop(opDiv, l, r) }
-                   | PCENT r:prefix             { l = newBinop(opMod, l, r) }
-                   )*                           { $$ = l }
+prod    = l:prefix ( STAR r:prefix              { l = newBinop(opMul, l, r) }
+                  | SLASH r:prefix              { l = newBinop(opDiv, l, r) }
+                  | PCENT r:prefix              { l = newBinop(opMod, l, r) }
+                  ) *                           { $$ = l }
 
 prefix  = PPLUS  p:prefix                       { $$ = newBinop(opPreAdd, lvalue(p), newInteger(1)) }
         | MMINUS p:prefix                       { $$ = newBinop(opPreSub, lvalue(p), newInteger(1)) }
@@ -3511,7 +3539,7 @@ params  = LPAREN p:mkobj
                          )
             )* RPAREN )                         { $$ = p }
 
-mkobj   =                                       { $$ = new(pObject) }
+mkobj   =                                       { $$ = (global new)(pObject) }
 
 primary = nil | number | string | symbol | var | lambda | subexpr | literal # | regex
 
@@ -3540,8 +3568,7 @@ number  = "-" n:unsign                          { $$ = neg(n) }
         | "+" n:number                          { $$ = n }
         |     n:unsign                          { $$ = n }
 
-unsign  = < DIGIT+ '.' DIGIT* EXP? > -          { $$ = newFloat(strtod(yytext, 0)) }
-        | < DIGIT* '.' DIGIT+ EXP? > -          { $$ = newFloat(strtod(yytext, 0)) }
+unsign  = < DIGIT* '.' DIGIT+ EXP? > -          { $$ = newFloat(strtod(yytext, 0)) }
         | "0" [bB] < BIGIT+ > -                 { $$ = newInteger(strtol(yytext, 0,  2)) }
         | "0" [xX] < HIGIT+ > -                 { $$ = newInteger(strtol(yytext, 0, 16)) }
         | "0" < OIGIT* > -                      { $$ = newInteger(strtol(yytext, 0,  8)) }
@@ -3550,11 +3577,16 @@ unsign  = < DIGIT+ '.' DIGIT* EXP? > -          { $$ = newFloat(strtod(yytext, 0
 
 string  = '"' < ( !'"' char )* > '"' -          { $$ = newStringUnescaped(yytext) }
 
-char    = "\\" ( ["'\\abfnrtv]
-               | [xX] HIGIT*
-               | [0-7][0-7]?[0-7]?
-               )
-        | .
+char           = '\\' [abefnrtv'"\[\]\\]
+               | '\\' [0-3][0-7][0-7]
+               | '\\' [xX] HIGIT*
+               | '\\' [0-7][0-7]?
+               | !'\\' .
+# char    = "\\" ( ["'\\abfnrtv]
+#                | [xX] HIGIT*
+#                | [0-7][0-7]?[0-7]?
+#                )
+#         | .
 
 symbol  = HASH i:id                             { $$ = i }
 
@@ -3669,7 +3701,7 @@ SLASH   = "/"  ![/=]    -
 SLASHEQ = "/="          -
 PCENT   = "%"  ![=]     -
 PCENTEQ = "%="          -
-DOT     = "."           -
+DOT     = "."  ![.]     -
 PLING   = "!"  ![=]     -
 TILDE   = "~"           -
 
@@ -3679,6 +3711,7 @@ xexpr   = expr | error @{ expected("expression", yytext) }
 
 %%
 ;
+#undef global
 
 #if PROFILE
 
@@ -3903,7 +3936,10 @@ oop eval(oop exp, oop env)
 # if PRIMCLOSURE
     if (Lambda == type) return newClosure(exp, env);
 # endif
-    if (Object != type) return exp;
+    if (Object != type) {
+        if (String == type) return newStringLen(_get(exp, String,value), _get(exp, String,length));
+        return exp;
+    }
     if (!opt_O) {
         Object_push(trace, exp);
         if (opt_d && opt_v) {
@@ -3948,6 +3984,22 @@ oop prim_Object_initialise(oop func, oop self, oop args, oop env)
     return self;
 }
 
+oop prim_newBinop(oop func, oop self, oop args, oop env)
+{                                                       assert(is(Object, args));
+    int argc     = _get(args, Object,isize);
+    if (argc != 3) fatal("newBinop: Expected 3 arguments, got %d\n", argc);
+    oop *indexed = _get(args, Object,indexed);
+    return newBinop(integerValue(indexed[0], "prim_newBinop"), indexed[1], indexed[2]);
+}
+
+oop prim_newApply(oop func, oop self, oop args, oop env)
+{                                                       assert(is(Object, args));
+    int argc     = _get(args, Object,isize);
+    if (argc != 2) fatal("newApply: Expected 2 arguments, got %d\n", argc);
+    oop *indexed = _get(args, Object,indexed);
+    return newApply(indexed[0], indexed[1]);
+}
+
 oop prim_Object_push(oop func, oop self, oop args, oop env)
 {                                                       assert(is(Object, args));
     int argc     = _get(args, Object,isize);            assert(is(Object, self));
@@ -4001,10 +4053,21 @@ oop prim_String_pop(oop func, oop self, oop args, oop env)
 }
 
 oop prim_String_asInteger(oop func, oop self, oop args, oop env)
-{                                                       assert(is(String, self));
+{                                                       assert(is(Object, args));
+    int argc     = _get(args, Object,isize);            assert(is(String, self));
+    if (argc > 1) fatal("String.asInteger: expected either 0 or 1 arguments, got %d\n", argc);
+
     char *str = String_content(self); // ensure nul terminator
     char *end = 0;
-    long value = strtol(str, &end, 0);
+    long value;
+
+    if (argc == 1) {
+        oop base = _get(args, Object,indexed)[0];
+        value = strtol(str, &end, integerValue(base, "String.asInteger"));
+    } else {
+        value = strtol(str, &end, 0);
+    }
+
     if (*end) return nil;
     return newInteger(value);
 }
@@ -4236,6 +4299,47 @@ oop prim_Symbol_asString(oop func, oop self, oop args, oop env)
     return newString(_get(self, Symbol,name));
 }
 
+
+oop prim_newStream(oop func, oop self, oop args, oop env)
+{                                                       assert(is(Object, args));    
+    int argc = _get(args, Object,isize);
+    if (argc != 1) fatal("newStream: Expected 1 argument, got %d\n", argc);
+    oop arg = _get(args, Object, indexed)[0];
+    if (!is(String, arg)) fatal("newStream: expected an argument of type String, got type %s instead\n", getType(arg));
+    return newStream(arg);
+}
+
+oop prim_Stream_atEnd(oop func, oop self, oop args, oop env)
+{
+    return newBoolean(_integerValue(Object_get(self, sym_position)) >= _integerValue(Object_get(self, sym_limit)));
+}
+
+oop prim_Stream_inc(oop func, oop self, oop args, oop env)
+{
+    if (_integerValue(Object_get(self, sym_position)) < _integerValue(Object_get(self, sym_limit))) {
+        // There has to be a better way of just adding 1
+        Object_put(self, sym_position, newInteger(_integerValue(Object_get(self, sym_position)) + 1));
+    }
+    return Object_get(self, sym_position);
+}
+
+oop prim_Stream_setLastBegin(oop func, oop self, oop args, oop env)
+{
+    return Object_put(self, sym_lastBegin, newInteger(_integerValue(Object_get(self, sym_position))));
+}
+
+oop prim_Stream_match(oop func, oop self, oop args, oop env)
+{                                                       assert(is(Object, args));    
+    int argc = _get(args, Object,isize);
+    if (argc != 1) fatal("Stream.match: Expected 1 argument, got %d\n", argc);
+    oop arg = _get(args, Object, indexed)[0];           assert(is(String, arg));
+    return newBoolean(strncmp(
+        String_content(Object_get(self, sym_content)) + _integerValue(Object_get(self, sym_position)),
+        String_content(arg),
+        strlen(String_content(arg))
+    ) == 0);
+}
+
 oop prim_length(oop func, oop self, oop args, oop env)
 {                                                       assert(is(Object, args));
     if (!is(Object, self)) valueError("length", "not an object", self);
@@ -4291,6 +4395,12 @@ oop prim_eval(oop func, oop self, oop args, oop env)
     int argc     = _get(args, Object,isize);
     oop *indexed = _get(args, Object,indexed);
     oop result   = nil;
+
+    //if (nil != Object_getLocal(args, sym_env)) {
+    if (Object_find(args, sym_env) >= 0) {
+        env = Object_getLocal(args, sym_env);
+    }
+
     for (int i = 0;  i < argc;  ++i) result = eval(indexed[i], env);
     return result;
 }
@@ -4300,6 +4410,22 @@ oop prim___eval__(oop func, oop self, oop args, oop env)
     return self;
 }
 
+oop prim_intern(oop func, oop self, oop args, oop env)
+{
+    int argc     = _get(args, Object,isize);
+    oop *indexed = _get(args, Object,indexed);
+    oop result   = nil;
+
+    if (argc != 1) {
+        fatal("intern: invalid number of arguments");
+    }
+    if (getType(indexed[0]) != String) {
+        fatal("intern: argument is not of type String, got %s instead", getTypeName(indexed[0]));
+    }
+
+    return intern(String_content(indexed[0]));
+}
+
 oop prim_print(oop func, oop self, oop args, oop env)
 {
     int argc     = _get(args, Object,isize);
@@ -4763,12 +4889,12 @@ oop applyThunkIn(oop func, oop env)
 typedef unsigned char byte;
 
 typedef enum op_t {
-     PUSH,      DROP,   POP,      DOT,     CLASS,   STRING,   RULE,   CALL,
+     PUSH,      DROP,   POP,      DOT,     CLASS,   STRING,   TEST,   RULE2,   RULE,   CALL,
      SUCCEED,   FAIL,   ACTION,   BEGIN,   END,     UNEND,    SET,
 } op_t;
 
 char *op_n[] = {
-    "PUSH",    "DROP", "POP",    "DOT",   "CLASS", "STRING", "RULE", "CALL",
+    "PUSH",    "DROP", "POP",    "DOT",   "CLASS", "STRING", "TEST", "RULE2", "RULE", "CALL",
     "SUCCEED", "FAIL", "ACTION", "BEGIN", "END",   "UNEND",  "SET",
 };
 
@@ -4779,8 +4905,9 @@ struct vmInsn {
         char   *str;
         oop     obj;
         vmInsn *code;
-    } arg;
-    unsigned short op, arglen, ok, ko;
+        int     len;
+    } arg, arg2;
+    unsigned short op, ok, ko;
 };
 
 typedef struct vmState
@@ -4789,7 +4916,7 @@ typedef struct vmState
     oop    variables;
 } vmState;
 
-#define VM_STATE_INITIALISER { nil, nil }
+#define VM_STATE_INITIALISER { nil, new(pObject) }
 
 void vmEnter(vmState *state, oop obj, char *yytext, int yyleng)
 {
@@ -4804,7 +4931,7 @@ void vmSet(vmState *state, oop obj, char *yytext, int yyleng)
 
 void vmAction(vmState *state, oop obj, char *yytext, int yyleng)
 {
-    oop text = yyleng ? newStringLen(yytext, yyleng) : nil;
+    oop text = yyleng >= 0 ? newStringLen(yytext, yyleng) : nil;
     Object_put(state->variables, sym_yytext, text);
     Object_put(state->variables, sym_yyleng, newInteger(yyleng));
     applyThunkIn(obj, state->variables);
@@ -4814,6 +4941,7 @@ void vmLeave(vmState *state, oop obj, char *yytext, int yyleng)
 {
     state->result = Object_getLocal(state->variables, sym_$$);
     state->variables = _getDelegate(state->variables);
+    Object_put(state->variables, sym_$$, state->result);
 }
 
 void vmDisassemble(vmInsn *code, int pc)
@@ -4824,15 +4952,15 @@ void vmDisassemble(vmInsn *code, int pc)
         case CLASS:
         case STRING:
             printf("%03d %-7s \"%s\" %2d %2d %2d\n",
-                   pc, op_n[i->op], i->arg.str, i->arglen, i->ok, i->ko);
+                   pc, op_n[i->op], i->arg.str, i->arg2.len, i->ok, i->ko);
             break;
         case CALL:
             printf("%03d %-7s %p %2d %2d %2d\n",
-                   pc, op_n[i->op], i->arg.code, i->arglen, i->ok, i->ko);
+                   pc, op_n[i->op], i->arg.code, i->arg2.len, i->ok, i->ko);
             break;
         default:
             printf("%03d %-7s %s %2d %2d %2d\n",
-                   pc, op_n[i->op], codeString(i->arg.obj, 0), i->arglen, i->ok, i->ko);
+                   pc, op_n[i->op], codeString(i->arg.obj, 0), i->arg2.len, i->ok, i->ko);
             break;
     }
 }
@@ -4867,18 +4995,27 @@ vmInsn *vmCompile(oop grammar, oop symbol)
     int     cpc  = 0;
     while (ppc < plen) {
         int op     = integerValue(prog[ppc++], "__match__");
-        oop arg    =              prog[ppc++];
-        int arglen = integerValue(prog[ppc++], "__match__");
+        oop arg    =              prog[ppc++];                          assert(arg);
+        oop arg2   =              prog[ppc++];                          assert(arg2);
         int ok     = integerValue(prog[ppc++], "__match__");
         int ko     = integerValue(prog[ppc++], "__match__");
         if (ok < 0 || ok >= clen) valueError("__match__", "OK destination out of range", program);
         if (ko < 0 || ko >= clen) valueError("__match__", "KO destination out of range", program);
-        code[cpc]  = (vmInsn){ .arg.obj = arg, .op = op, .arglen = arglen, .ok = ok, .ko = ko };
+        code[cpc]  = (vmInsn){ .arg.obj = arg, .arg2.obj = arg2, .op = op, .ok = ok, .ko = ko };
         switch (op) {
             case CLASS:
             case STRING: {
-                code[cpc].arglen  = stringLength(code[cpc].arg.obj, "__match__");
-                code[cpc].arg.str = stringValue (code[cpc].arg.obj, "__match__");
+                code[cpc].arg2.len = stringLength(code[cpc].arg.obj, "__match__");
+                code[cpc].arg.str  = stringValue (code[cpc].arg.obj, "__match__");
+                break;
+            }
+            case TEST: {
+                if (!isClosure(code[cpc].arg.obj)) valueError("__match__", "TEST argument must be a closure", program);
+                break;
+            }
+            case RULE2: {
+                if (!is(Symbol, code[cpc].arg.obj )) valueError("__match__", "RULE2 argument must be a symbol", program);
+                if (!is(Object, code[cpc].arg2.obj)) valueError("__match__", "RULE2 argument2 must be an object", program);
                 break;
             }
             case RULE: {
@@ -4952,6 +5089,7 @@ int vmRun(oop grammar, oop symbol, char *text, int start, int length)
     context.nactions = 0;
 
     vmState state = VM_STATE_INITIALISER;
+    state.variables = new(pObject);
 
 # define saveAction(ACT, OBJ, BEG, LEN) {                                       \
         if (context.nactions >= maxactions)                                     \
@@ -4973,7 +5111,6 @@ int vmRun(oop grammar, oop symbol, char *text, int start, int length)
     frame.pc       = 0;
     frame.nactions = context.nactions;
 
-
     int textbeg = 0, textend = 0;
     int result = 0;
 
@@ -4992,6 +5129,8 @@ int vmRun(oop grammar, oop symbol, char *text, int start, int length)
         C##stack[--C##sp];                      \
     })
 
+    saveAction(vmEnter, nil, 0, 0);
+    
     for (;;) {
         if (opt_d) vmDisassemble(frame.code, frame.pc);
         vmInsn *i = frame.code + frame.pc++;
@@ -5021,9 +5160,9 @@ int vmRun(oop grammar, oop symbol, char *text, int start, int length)
                 continue;
             }
             case STRING: {
-                if (context.position + i->arglen <= length) {
-                    if (0 == memcmp(text + context.position, i->arg.str, i->arglen)) {
-                        context.position += i->arglen;
+                if (context.position + i->arg2.len <= length) {
+                    if (0 == memcmp(text + context.position, i->arg.str, i->arg2.len)) {
+                        context.position += i->arg2.len;
                         frame.pc = i->ok;
                         continue;
                     }
@@ -5031,6 +5170,16 @@ int vmRun(oop grammar, oop symbol, char *text, int start, int length)
                 frame.pc = i->ko;
                 continue;
             }
+            case TEST: {
+                oop result = apply(i->arg.obj, nil, new(pObject), nil, nil);
+                frame.pc = (nil == result) ? i->ko : i->ok;
+                continue;
+            }
+            case RULE2: {
+                i->op = CALL;
+                i->arg.code = vmCacheGet(i->arg2.obj, i->arg.obj);
+                goto doCall;
+            }
             case RULE: {
                 // frame.pc--; // save pc of call insn
                 // push(r, frame);
@@ -5073,11 +5222,15 @@ int vmRun(oop grammar, oop symbol, char *text, int start, int length)
             }
             case FAIL: {
                 if (rsp) {
+                    context.nactions = frame.nactions - 1; // remove all actions added by this rule
                     frame = pop(r);
                     i = frame.code + frame.pc;
                     frame.pc = i->ko;
                     continue;
                 }
+                else {
+                    context.nactions = 0;
+                }
                 result = -1;
                 break;
             }
@@ -5115,8 +5268,6 @@ int vmRun(oop grammar, oop symbol, char *text, int start, int length)
         break;
     }
 
-    saveAction(vmLeave, nil, 0, 0);
-
 #undef pop
 #undef drop
 #undef push
@@ -5153,6 +5304,20 @@ oop prim_match(oop func, oop self, oop args, oop env)
 
 #endif // PEGVM
 
+oop prim_lvalue(oop func, oop self, oop args, oop env)
+{                                                               assert(is(Object, args));
+    int argc = _get(args, Object,isize);
+    if (argc != 1) fatal("lvalue: one argument expected\n");
+    return lvalue(_get(args, Object,indexed)[0]);
+}
+
+oop prim_assign(oop func, oop self, oop args, oop env)
+{                                                               assert(is(Object, args));
+    int argc = _get(args, Object,isize);
+    if (argc != 2) fatal("assign: 2 arguments expected\n");
+    return assign(_get(args, Object,indexed)[0], _get(args, Object,indexed)[1]);
+}
+
 oop replFile(FILE *in)
 {
     int oldline = lineno;
@@ -5278,6 +5443,18 @@ int main(int argc, char **argv)
 
     Object_put(pObject, prop_eval, newPrimitive(prim___eval__, newString("Object.__eval__")));        // inherited by all objects
 
+#define stringify(x) #x
+
+#define declareOp(NAME, OP) _set(intern(stringify(op##NAME)), Symbol,value, newInteger(op##NAME));
+    doBinops(declareOp)
+#undef declareOp
+
+#undef stringify
+
+#define declareOp(NAME, OP) _set(intern(#NAME), Symbol,value, newInteger(NAME));
+    doUnyops(declareOp)
+#undef declareOp
+
 #if TYPECODES
 
 # define defineEvaluator(NAME)                                          \
@@ -5323,6 +5500,12 @@ int main(int argc, char **argv)
     prim(defined    , prim_defined);
     prim(__extern__ , prim_extern);
     prim(__match__  , prim_match);
+    prim(intern     , prim_intern);
+    prim(newBinop   , prim_newBinop);
+    prim(newApply   , prim_newApply);
+    prim(lvalue     , prim_lvalue);
+    prim(assign     , prim_assign);
+    prim(newStream  , prim_newStream);
 
 # undef prim
 
@@ -5365,6 +5548,10 @@ int main(int argc, char **argv)
     method(Symbol,define,       prim_Symbol_define      );
     method(Symbol,value,        prim_Symbol_value       );
     method(Symbol,allInstances, prim_Symbol_allInstances);
+    method(Stream,atEnd,        prim_Stream_atEnd    );
+    method(Stream,inc,          prim_Stream_inc      );
+    method(Stream,setLastBegin, prim_Stream_setLastBegin);
+    method(Stream,match,        prim_Stream_match    );
 
 # undef method
 

rawMetaGrammar.leg

@@ -0,0 +1,73 @@
+statement  = b:block                                         { b; }
+           | e:expression SEMI                               { e; }
+
+expression = p:primary
+           ( DOT   i:id               ASSIGN e:expression  # { $$ = newSetProp(p, i, e) }
+           | LBRAK i:expression RBRAK ASSIGN e:expression    { SetArray.new(object: p, index: i, value: e); }
+           )
+           | i:id                     ASSIGN e:expression    { SetVar.new(name: i, value: e); }
+           | pf:postfix                                      { pf; }
+
+postfix    = p:primary
+           ( DOT   i:id a:args  !ASSIGN !LBRACE              { p = Invoke.new(self: p, method: i, arguments: a); }
+           | DOT   i:id         !ASSIGN                      { p = GetProp.new(object: p, key: i); }
+           | a:args !ASSIGN !LBRACE                          { p = Call.new(function: p, arguments: a); }
+           ) *                                               { p; }
+
+args       = LPAREN a:mklist
+           (
+           ( k:id COLON e:expression                         { a[k] = e; }
+           | e:expression                                    { a.push(e); }
+           )
+           ( COMMA
+           ( k:id COLON e:expression                         { a[k] = e; }
+           | e:expression                                    { a.push(e); }
+           )
+           ) *
+           ) ? RPAREN                                        { a; }
+
+mklist     =                                                 { Object.new(); }
+
+primary    = nil | number | var | subExpr
+
+subExpr    = LPAREN e:expression RPAREN                      { e; }
+
+block      = LBRACE b:mklist
+           ( e:statement                                     { b.push(e); }
+           ) * RBRACE                                        { Block.new(body: b); }
+
+nil        = NIL                                             { nil; }
+
+number  = "-" n:unsign                                       { Unyop.new(operation: __opNeg).push(n) }
+        | "+" n:number                                       { n }
+        |     n:unsign                                       { n }
+
+unsign  = < DIGIT+ > -                                       { yytext.asInteger(10); }
+
+var        = i:id                                            { GetVar.new(name: i); }
+
+id         = < LETTER ALNUM* > -                             { intern(yytext); }
+
+DIGIT          = [0-9]
+LETTER         = [A-Za-z_]
+ALNUM          = LETTER | DIGIT
+
+NIL            = "nil" !ALNUM -
+
+SEMI           = ";"          -
+COMMA          = ","          -
+LPAREN         = "("          -
+RPAREN         = ")"          -
+LBRAK          = "["          -
+RBRAK          = "]"          -
+LBRACE         = "{"          -
+RBRACE         = "}"          -
+COLON          = ":" ![:]     -
+ASSIGN         = "=" ![=]     -
+DOT            = "." ![.]     -
+
+-              = ( space | comment )*
+space          = ' ' | '\t' | end-of-line
+comment        = '//' ( !end-of-line . )* end-of-line
+end-of-line    = '\r\n' | '\n' | '\r'
+end-of-file    = !.

rawgrammar.leg

@@ -0,0 +1,79 @@
+grammar        = - ( d:definition                        { d; }
+                   | end-of-file                         { nil; }
+                   )
+
+definition     = i:identifier ASSIGN e:expression SEMI?  { Definition.new(name: i, expression: e); }
+
+expression     = s:sequence !BAR                         { s; }
+               | s1:sequence                             { s1 = Alternation.new().push(s1); }
+                 ( BAR s2:sequence                       { s1.push(s2); }
+                 ) *                                     { s1; }
+
+sequence       = p:prefix ( q:prefix                     { p = Sequence.new().push(p).push(q); }
+               ( q:prefix                                { p.push(q); }
+               ) * ) ?                                   { p; }
+
+prefix         = AND a:action                            { ParseTimeAction.new(action: a); }
+               | AT  a:action                            { ExecuteAction.new(action: a); }
+               |
+                 ( AND s:suffix                          { And.new(expression: s); }
+                 | NOT s:suffix                          { Not.new(expression: s); }
+                 | s:suffix                              { s; }
+                 )
+
+suffix         = p:primary
+               ( QUERY                                   { p = Optional.new(expression: p); }
+               | STAR                                    { p = Star.new(expression: p); }
+               | PLUS                                    { p = Plus.new(expression: p); }
+               ) ?                                       { p; }
+
+primary        = i1:identifier COLON i2:ruleCall !ASSIGN { Assignment.new(name: i1, rule: i2); }
+               | i:ruleCall !ASSIGN                      { i; }
+               | LPAREN e:expression RPAREN              { e; }
+               | l:literal                               { l; }
+               | c:class                                 { c; }
+               | DOT                                     { Dot.new(); }
+               | a:action                                { a; }
+               | BEGIN e:expression END                  { Capture.new(expression: e); }
+
+identifier     = < [-a-zA-Z_][-a-zA-Z_0-9]* > -          { intern(yytext); }
+
+ruleCall       = n:identifier CCOLON r:identifier        { NamespacedRuleCall.new(namespace: n, name: r); }
+               | i:identifier                            { RuleCall.new(name: i); }
+
+literal        = ['] < ( !['] char )* > ['] -            { StringLiteral.new(string: yytext); }
+               | ["] < ( !["] char )* > ["] -            { StringLiteral.new(string: yytext); }
+
+class          = '[' < ( !']' range )* > ']' -           { CharacterClass.new(value: yytext); }
+
+range          = char '-' char | char
+
+char           = '\\' [abefnrtv'"\[\]\\]
+               | '\\' [0-3][0-7][0-7]
+               | '\\' [0-7][0-7]?
+               | !'\\' .
+
+action         = m:metaLanguage::block -                 { Action.new(parseTree: Block.new(body: m)); }
+
+-              = ( space | comment )*
+space          = ' ' | '\t' | end-of-line
+comment        = '#' ( !end-of-line . )* end-of-line
+end-of-line    = '\r\n' | '\n' | '\r'
+end-of-file    = !.
+
+BAR            = '|'          -
+NOT            = '!'          -
+QUERY          = '?'          -
+BEGIN          = '<'          -
+END            = '>'          -
+SEMI           = ";"          -
+CCOLON         = "::"         -
+LPAREN         = "("          -
+RPAREN         = ")"          -
+ASSIGN         = "=" ![=]     -
+AND            = "&" ![&=]    -
+PLUS           = "+" ![+=]    -
+STAR           = "*" ![=]     -
+DOT            = "." ![.]     -
+COLON          = ":" ![:]     -
+AT             = "@"          -

rawminproto.leg

@@ -0,0 +1,275 @@
+start   = - ( s:stmt                            { s; }
+            | !.                                # { yysval = 0 }
+            | < (!EOL .)* >                     # { fatal("syntax error near: %s", yytext) }
+            )
+
+stmt    = WHILE LPAREN c:expr RPAREN s:stmt     { While.new(condition: c, body: s) }
+        | IF LPAREN c:expr RPAREN s:stmt
+          ( ELSE t:stmt                         { If.new(condition: c, consequent: s, alternate: t  ) }
+          |                                     { If.new(condition: c, consequent: s, alternate: nil) }
+          )
+        | CONT EOS                              { Continue.new() }
+        | BREAK e:expr EOS                      { Break.new(value: e) }
+        | BREAK EOS                             { Break.new(value: nil) }
+        | RETURN e:expr EOS                     { Return.new(value: e) }
+        | RETURN EOS                            { Return.new(value: nil) }
+        | FOR LPAREN i:id IN e:expr RPAREN
+              s:stmt                            { ForIn.new(identifier: i, expression: e, body: s) }
+        | FOR LPAREN i:id FROM a:expr
+                 TO  b:expr RPAREN s:stmt       { ForFromTo.new(identifier: i, first: a, last: b, body: s) }
+        | FOR LPAREN i:expr SEMI c:expr SEMI
+                     u:expr RPAREN s:stmt       { For.new(initialise: i, condition: c, update: u, body: s) }
+        | TRY t:stmt
+          ( CATCH LPAREN i:id RPAREN c:stmt     { TryCatch.new(statement: t, identifier: i, handler: c) }
+          | ENSURE e:stmt                       { TryEnsure.new(statement: t, handler: e) }
+          )
+        | RAISE e:expr EOS                      { Raise.new(value: e) }
+        | LOCAL       i:id p:params b:block     { SetLocal.new(name: i, value: Lambda.new(parameters: p, body: b)) }
+        | GLOBAL      i:id p:params b:block     { SetGlobal.new(name: i, value: Lambda.new(parameters: p, body: b)) }
+        |             i:id p:params b:block     { SetVar.new(name: i, value: Lambda.new(parameters: p, body: b)) }
+        | v:proto DOT i:id p:params b:block     { SetProp.new(object: v, key: i, value: Lambda.new(parameters: p, body: b)) }
+        | b:block                               { Block.new(body: b) }
+        | e:expr EOS                            { e }
+
+proto   = v:var ( DOT j:id !LPAREN              { v = GetProp.new(object: v, key: j) }
+                )*                              { v }
+
+EOS     = SEMI+ | &RBRACE | &ELSE | &CATCH
+
+expr    = LOCAL  i:id ASSIGN e:expr             { SetLocal.new(name: i, value: e) }
+        | GLOBAL i:id ASSIGN e:expr             { SetGlobal.new(name: i, value: e) }
+        |        i:id ASSIGN e:expr             { SetVar.new(name: i, value: e) }
+        | l:logor ( ASSIGN   r:expr             { l  = assign(l, r) }
+                  | PLUSEQ   r:expr             { l  = newBinop(__opPreAdd, lvalue(l), r) }
+                  | MINUSEQ  r:expr             { l  = newBinop(__opPreSub, lvalue(l), r) }
+                  | STAREQ   r:expr             { l  = newBinop(__opPreMul, lvalue(l), r) }
+                  | SLASHEQ  r:expr             { l  = newBinop(__opPreDiv, lvalue(l), r) }
+                  | PCENTEQ  r:expr             { l  = newBinop(__opPreMod, lvalue(l), r) }
+                  | SHLEQ    r:expr             { l  = newBinop(__opPreShl, lvalue(l), r) }
+                  | SHREQ    r:expr             { l  = newBinop(__opPreShr, lvalue(l), r) }
+                  | ANDEQ    r:expr             { l  = newBinop(__opPreAnd, lvalue(l), r) }
+                  | XOREQ    r:expr             { l  = newBinop(__opPreXor, lvalue(l), r) }
+                  | OREQ     r:expr             { l  = newBinop(__opPreOr,  lvalue(l), r) }
+                  )?                            { l }
+
+logor   = l:logand ( BARBAR r:logand            { l = newBinop(__opLogOr,  l, r) }
+                   )*                           { l }
+
+logand  = l:bitor ( ANDAND r:bitor              { l = newBinop(__opLogAnd, l, r) }
+                  )*                            { l }
+
+bitor   = l:bitxor ( OR r:bitxor                { l = newBinop(__opBitOr,  l, r) }
+                   )*                           { l }
+
+bitxor  = l:bitand ( XOR r:bitand               { l = newBinop(__opBitXor, l, r) }
+                   )*                           { l }
+
+bitand  = l:eq ( AND r:eq                       { l = newBinop(__opBitAnd, l, r) }
+               )*                               { l }
+
+eq      = l:ineq ( EQ    r:ineq                 { l = newBinop(__opEq,    l, r) }
+                 | NOTEQ r:ineq                 { l = newBinop(__opNotEq, l, r) }
+                 )*                             { l }
+
+ineq    = l:shift ( LESS   r:shift              { l = newBinop(__opLess,   l, r) }
+                  | LESSEQ r:shift              { l = newBinop(__opLessEq, l, r) }
+                  | GRTREQ r:shift              { l = newBinop(__opGrtrEq, l, r) }
+                  | GRTR   r:shift              { l = newBinop(__opGrtr,   l, r) }
+                  )*                            { l }
+
+shift   = l:sum ( SHL r:sum                     { l = newBinop(__opShl, l, r) }
+                | SHR r:sum                     { l = newBinop(__opShr, l, r) }
+                )*                              { l }
+
+sum     = l:prod ( PLUS  r:prod                 { l = newBinop(__opAdd, l, r) }
+                 | MINUS r:prod                 { l = newBinop(__opSub, l, r) }
+                 )*                             { l }
+
+prod    = l:range ( STAR  r:range               { l = newBinop(__opMul, l, r) }
+                  | SLASH r:range               { l = newBinop(__opDiv, l, r) }
+                  | PCENT r:range               { l = newBinop(__opMod, l, r) }
+                  ) *                           { l }
+
+range   = i1:prefix ( DOTDOT i2:prefix          { i1 = Range.new(start: i1, end: i2) }
+        ) ?                                     { i1 }
+
+prefix  = PPLUS  p:prefix                       { newBinop(__opPreAdd, lvalue(p), 1) }
+        | MMINUS p:prefix                       { newBinop(__opPreSub, lvalue(p), 1) }
+        | PLING  p:prefix                       { Unyop.new(operation: __opNot).push(p) }
+        | MINUS  p:prefix                       { Unyop.new(operation: __opNeg).push(p) }
+        | TILDE  p:prefix                       { Unyop.new(operation: __opCom).push(p) }
+        | BQUOTE s:stmt                         { Unyop.new(operation: __opQuasiquote).push(s) }
+        | COMMAT e:expr                         { Unyop.new(operation: __opUnquote).push(e) }
+        | postfix
+
+postfix = SUPER DOT i:id a:args                 { Super.new(method: i, arguments: a) }
+        | p:primary
+          ( LBRAK e:expr RBRAK                  { p = GetArray.new(object: p, index: e) }
+          | DOT   i:id ( a:args !LBRACE         { p = Invoke.new(self: p, method: i, arguments: a) }
+                       |                        { p = GetProp.new(object: p, key: i) }
+                       )
+          |              a:args !LBRACE         { p = newApply(p, a) }
+          )*
+          ( PPLUS                               { p  = newBinop(__opPostAdd, lvalue(p),  1) }
+          | MMINUS                              { p  = newBinop(__opPostAdd, lvalue(p), -1) }
+          )?                                    { p }
+
+args    = LPAREN a:mkobj
+          ( RPAREN
+          | ( k:id COLON e:expr                 { a[k] = e }
+            | e:expr                            { a.push(e) }
+            )
+            ( COMMA ( k:id COLON e:expr         { a[k] = e }
+                    | e:expr                    { a.push(e) }
+                    ) )* RPAREN )               { a }
+
+params  = LPAREN p:mkobj
+          ( RPAREN
+          | i:id ( COLON e:expr                 { p[i] = e }
+                 |                              { p.push(i) }
+                 )
+            ( COMMA i:id ( COLON e:expr         { p[i] = e }
+                         |                      { p.push(i) }
+                         )
+            )* RPAREN )                         { p }
+
+mkobj   =                                       { Object.new() }
+
+primary = nil | number | string | symbol | var | lambda | subexpr | literal
+
+lambda  = p:params b:block                      { Lambda.new(parameters: p, body: b) }
+
+subexpr = LPAREN e:expr  RPAREN                 { e }
+        |        b:block                        { Block.new(body: b) }
+
+literal = LBRAK o:mkobj
+          ( RBRAK
+          | ( ( i:id COLON e:expr               { o[i] = e }
+              | e:expr                          { o.push(e) }
+              ) ( COMMA ( i:id COLON e:expr     { o[i] = e }
+                        | e:expr                { o.push(e) }
+                        ) )* )? RBRAK )         { Literal.new(object: o) }
+
+block   = LBRACE b:mkobj
+            (  e:stmt                           { b.push(e) }
+            )* RBRACE                           { b }
+
+nil     = NIL                                   { nil }
+
+number  = "-" n:unsign                          { Unyop.new(operation: __opNeg).push(n) }
+        | "+" n:number                          { n }
+        |     n:unsign                          { n }
+
+unsign  = < DIGIT* '.' DIGIT+ EXP? > -          { yytext.asFloat() }
+        | "0" [bB] < BIGIT+ > -                 { yytext.asInteger(2) }
+        | "0" [xX] < HIGIT+ > -                 { yytext.asInteger(16) }
+        | "0" < OIGIT* > -                      { yytext.asInteger(8) }
+        | < DIGIT+ > -                          { yytext.asInteger() }
+        | "'" < char > "'" -                    { ord(yytext.unescaped()) }
+
+string  = '"' < ( !'"' char )* > '"' -          { yytext }
+
+# Version originale, qui ne parse pas deux antislash pour une raison obscure
+# char    = "\\" ( ["'\\abfnrtv]
+#                | [xX] HIGIT*
+#                | [0-7][0-7]?[0-7]?
+#                )
+#         | .
+
+# Version de rawgrammar.leg, qui fonctionne sans problème
+char           = '\\' [abefnrtv'"\[\]\\]
+               | '\\' [0-3][0-7][0-7]
+               | '\\' [0-7][0-7]?
+               | '\\' [xX] HIGIT+
+               | !'\\' .
+
+symbol  = HASH i:id                             { i }
+
+var     = LOCAL  i:id                           { GetLocal.new(name: i) }
+        | GLOBAL i:id                           { GetGlobal.new(name: i) }
+        |        i:id                           { GetVar.new(name: i) }
+
+id      = < LETTER ALNUM* > -                   { intern(yytext) }
+
+BIGIT   = [0-1]
+OIGIT   = [0-7]
+DIGIT   = [0-9]
+HIGIT   = [0-9A-Fa-f]
+LETTER  = [A-Za-z_$?]
+ALNUM   = LETTER | DIGIT
+SIGN    = [-+]
+EXP     = [eE] SIGN DIGIT+
+
+-       = SPACE*
+
+SPACE   = [ \t] | EOL | SLC | MLC
+EOL     = [\n\r]                                # { ++lineno }
+SLC     = "//" (!EOL .)*
+MLC     = "/*" ( MLC | !"*/" (EOL | .))* "*/" -
+
+NIL     = "nil"      !ALNUM  -
+WHILE   = "while"    !ALNUM  -
+IF      = "if"       !ALNUM  -
+ELSE    = "else"     !ALNUM  -
+FOR     = "for"      !ALNUM  -
+IN      = "in"       !ALNUM  -
+FROM    = "from"     !ALNUM  -
+TO      = "to"       !ALNUM  -
+CONT    = "continue" !ALNUM  -
+BREAK   = "break"    !ALNUM  -
+RETURN  = "return"   !ALNUM  -
+TRY     = "try"      !ALNUM  -
+CATCH   = "catch"    !ALNUM  -
+ENSURE  = "ensure"   !ALNUM  -
+RAISE   = "raise"    !ALNUM  -
+GLOBAL  = "global"   !ALNUM  -
+LOCAL   = "local"    !ALNUM  -
+SUPER   = "super"    !ALNUM  -
+
+BQUOTE  = "`"           -
+COMMAT  = "@"           -
+HASH    = "#"           -
+SEMI    = ";"           -
+ASSIGN  = "="  ![=]     -
+COMMA   = ","           -
+COLON   = ":"  ![:]     -
+LPAREN  = "("           -
+RPAREN  = ")"           -
+LBRAK   = "["           -
+RBRAK   = "]"           -
+LBRACE  = "{"           -
+RBRACE  = "}"           -
+BARBAR  = "||" ![=]     -
+ANDAND  = "&&" ![=]     -
+OR      = "|"  ![|=]    -
+OREQ    = "|="          -
+XOR     = "^"  ![=]     -
+XOREQ   = "^="          -
+AND     = "&"  ![&=]    -
+ANDEQ   = "&="          -
+EQ      = "=="          -
+NOTEQ   = "!="          -
+LESS    = "<"  ![<=]    -
+LESSEQ  = "<="          -
+GRTREQ  = ">="          -
+GRTR    = ">"  ![=]     -
+SHL     = "<<" ![=]     -
+SHLEQ   = "<<="         -
+SHR     = ">>" ![=]     -
+SHREQ   = ">>="         -
+PLUS    = "+"  ![+=]    -
+PLUSEQ  = "+="          -
+PPLUS   = "++"          -
+MINUS   = "-"  ![-=]    -
+MINUSEQ = "-="          -
+MMINUS  = "--"          -
+STAR    = "*"  ![=]     -
+STAREQ  = "*="          -
+SLASH   = "/"  ![/=]    -
+SLASHEQ = "/="          -
+PCENT   = "%"  ![=]     -
+PCENTEQ = "%="          -
+DOT     = "."  ![.]     -
+DOTDOT  = ".."          -
+PLING   = "!"  ![=]     -
+TILDE   = "~"           -

test.txt

@@ -378,6 +378,12 @@ ast = (`6 * @myNode;); // user-defined node in AST
 
 print("AST eval => ", eval(ast), "\n");
 
+print("String to int conversion\n");
+print("11111111".asInteger(2), "\n");
+print("377".asInteger(8), "\n");
+print("255".asInteger(), "\n");
+print("FF".asInteger(16), "\n");
+
 for (i from 0 to 10) 		     print(i, " ");  print("\n");
 for (i from 10 to 0) 		     print(i, " ");  print("\n");
 for (i in 10)        		     print(i, " ");  print("\n");