Mercurial > hg > Members > kono > PLparser
changeset 7:619472ca4742
Refactoring for multiple implementation of scanner
author | one |
---|---|
date | Wed, 01 Sep 2010 15:23:37 +0900 |
parents | 563bcb96e4fa |
children | 8d0f9c1816f5 |
files | src/plparser/PLScanner.java src/plparser/PLScannerImpl.java src/plparser/PropertyListParser.java src/plparser/PropertyListScanner.java src/plparser/TestScanner.java |
diffstat | 5 files changed, 183 insertions(+), 124 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plparser/PLScanner.java Wed Sep 01 15:23:37 2010 +0900 @@ -0,0 +1,39 @@ +package plparser; + +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.InputStream; +import java.io.InputStreamReader; + +public interface PLScanner<Node> { + + String filename(); + int lineno(); + + public Token<Node> nextToken(); + + public boolean hasRemaining(); + + public PLScanner<Node> pushScannerFile(InputStream newfile, String prompt); + PLScanner<Node> prev(); + PLScanner<Node> pushScanner(String exp); + PLScanner<Node> pushScannerFile(String file) throws FileNotFoundException; + PLScanner<Node> popScanner(); + public PLScanner<Node> set(String exp); + + /* + * Read From File + * We cannot read symbol bigger than Buffersize + */ + public PLScanner<Node> setFile(String file) + throws FileNotFoundException; + + public PLScanner<Node> set(InputStreamReader file); + + public void error(String err); + + public Iterable<Token<Node>> scanToken(String exp); + + public Iterable<Token<Node>> scanToken(FileReader file); + +} \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plparser/PLScannerImpl.java Wed Sep 01 15:23:37 2010 +0900 @@ -0,0 +1,104 @@ +package plparser; + +import java.io.FileReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Iterator; + +public abstract class PLScannerImpl<Node> implements PLScanner<Node> { + + public int lineno; + public Token<Node> nextToken; + public Dictionary<Node> dict; + public PLScanner<Node> next; + public InputStreamReader file; + public String filename; + public PLScanner<Node> prev; + public Token<Node> nullToken; + public String prompt; + public static final int BufferSize = 4096; + + public PLScannerImpl() { + super(); + } + + public PLScannerImpl(PLScannerImpl<Node> plScannerImpl, + Dictionary<Node> dict, Token<Node> nullToken) { + this.dict = dict; + this.nullToken = nullToken; + } + + + public PLScanner<Node> popScanner() { + return prev; + } + + private PLScanner<Node> findFileName() { + for(PLScanner<Node> s = this;s!=null ; s = s.prev()) { + if (s.filename()!=null) return s; + } + return null; + } + /* + * Read From String + */ + + public void error(String err) { + PLScanner<Node> s = findFileName(); + if (s!=null) { + System.err.print(s.filename()+":"+s.lineno()+": "); + } + System.err.println("error: "+err); + } + + public Iterable<Token<Node>> scanToken(String exp) { + set(exp); + return iterator(); + } + + public Iterable<Token<Node>> scanToken(FileReader file) { + set(file); + return iterator(); + } + + private Iterable<Token<Node>> iterator() { + return new Iterable<Token<Node>>() { + public Iterator<Token<Node>> iterator() { + return new Iterator<Token<Node>>() { + public boolean hasNext() { + return hasRemaining(); + } + public Token<Node> next() { + return nextToken(); + } + public void remove() { + } + }; + } + }; + } + + public PLScanner<Node> setFile(InputStream newfile, String prompt) { + this.filename = newfile.toString(); + nextToken = nullToken; + this.prompt = prompt; + set(new InputStreamReader(newfile)); + return this; + } + + @Override + public String filename() { + return filename; + } + + @Override + public int lineno() { + return lineno; + } + + @Override + public PLScanner<Node> prev() { + return prev; + } + +} \ No newline at end of file
--- a/src/plparser/PropertyListParser.java Mon Aug 30 12:35:23 2010 +0900 +++ b/src/plparser/PropertyListParser.java Wed Sep 01 15:23:37 2010 +0900 @@ -8,7 +8,7 @@ public class PropertyListParser<Node extends Property> { PropertyListNodeFactory<Node> lf; Token<Node> nextToken; - public PropertyListScanner<Node> scanner; + public PLScanner<Node> scanner; private Dictionary<Node> dict; // scope is necessary if you have to parse nested name scope // private PropertyListScope<Node> scope; @@ -49,7 +49,6 @@ public Node parse() { if (scanner==null) return null; // internal error - if (scanner.cb==null) return null; // nothing to do nextToken(); return term(); } @@ -59,7 +58,7 @@ scanner = scanner.pushScanner(exp); n = parse(); scanner = scanner.popScanner(); - nextToken = scanner.nextToken; + nextToken = scanner.nextToken(); return n; } @@ -90,7 +89,7 @@ n=parse(); } while(scanner.hasRemaining()); scanner = scanner.popScanner(); - nextToken = scanner.nextToken; + nextToken = scanner.nextToken(); return n; }
--- a/src/plparser/PropertyListScanner.java Mon Aug 30 12:35:23 2010 +0900 +++ b/src/plparser/PropertyListScanner.java Wed Sep 01 15:23:37 2010 +0900 @@ -6,11 +6,10 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.CharBuffer; -import java.util.Iterator; import java.util.regex.Matcher; import java.util.regex.Pattern; -public class PropertyListScanner<Node> { +public class PropertyListScanner<Node> extends PLScannerImpl<Node> implements PLScanner<Node> { // We cannot make Generic Singleton pattern // static PropertyListScanner scanner = new PropertyListScanner(); @@ -31,18 +30,8 @@ * 2010/8 Shinji Kono */ - public int lineno; public Matcher scan; - public Token<Node> nextToken; - public Dictionary<Node> dict; - public PropertyListScanner<Node> next; - protected CharBuffer cb; - private InputStreamReader file; - private String filename; - public PropertyListScanner<Node> prev; - public Token<Node> nullToken ; - public String prompt; - + private CharBuffer cb; public PropertyListScanner(Dictionary<Node> dict) { this.dict = dict; nullToken = new Token<Node>("",TokenID.NULL); @@ -59,44 +48,28 @@ // Pattern must contain exact 1 group - public static Pattern tokenPat = Pattern.compile( + private static Pattern tokenPat = Pattern.compile( "([={}(),;])" ); - public static Pattern namePat = Pattern.compile("([_a-zA-Z][\\@\\w]*)"); - public static final Pattern numPat = Pattern.compile("([0-9]+)"); - public static final Pattern stringPat1 = Pattern.compile("\\\"([^\"]*)\\\""); - public static final Pattern stringPat = Pattern.compile("\\'([^\\']*)\\'"); - public static final Pattern stringPat1cont = Pattern.compile("\\\"([^\"]*)$"); - public static final Pattern stringPatCont = Pattern.compile("\\'([^\\']*)$"); - public static final Pattern stringPat1End = Pattern.compile("([^\"]*)\\\""); - public static final Pattern stringPatEnd = Pattern.compile("([^\\']*)\'"); - public static final Pattern commentPat = Pattern.compile("(//.*)"); - public static final Pattern commentPat1 = Pattern.compile("(/\\*)"); - public static final Pattern commentPat1End = Pattern.compile("(.*\\*/)"); - public static final Pattern errorPat = Pattern.compile("([^\\s])"); - public static final Pattern anyPat = Pattern.compile("(.)"); - private static final int BufferSize = 4096; - - /* - * Get next token - * - * No looking up method nor put back. It never returns null but - * may return nullToken. So nextToken.type is always valid. - * nullToken means the end of the input. - * - * Token is a syntax element and it may have macro binding as - * predicate, infix or prefix operator. To get the value, use - * makeVariable(). Operator order for infix and prefix is in - * Token.order. TokenID.order is default order for fix element and - * currently never used. - * - * When matcher hit an end of the input, hasRemaining() method try - * to extend the input using extendInput(). - */ - + private static Pattern namePat = Pattern.compile("([_a-zA-Z][\\@\\w]*)"); + private static final Pattern numPat = Pattern.compile("([0-9]+)"); + private static final Pattern stringPat1 = Pattern.compile("\\\"([^\"]*)\\\""); + private static final Pattern stringPat = Pattern.compile("\\'([^\\']*)\\'"); + private static final Pattern stringPat1cont = Pattern.compile("\\\"([^\"]*)$"); + private static final Pattern stringPatCont = Pattern.compile("\\'([^\\']*)$"); + private static final Pattern stringPat1End = Pattern.compile("([^\"]*)\\\""); + private static final Pattern stringPatEnd = Pattern.compile("([^\\']*)\'"); + private static final Pattern commentPat = Pattern.compile("(//.*)"); + private static final Pattern commentPat1 = Pattern.compile("(/\\*)"); + private static final Pattern commentPat1End = Pattern.compile("(.*\\*/)"); + private static final Pattern errorPat = Pattern.compile("([^\\s])"); + private static final Pattern anyPat = Pattern.compile("(.)"); + + @Override public Token<Node> nextToken() { String s; nextToken = nullToken; + if (cb==null) return nextToken; while(hasRemaining()) { scan.reset(); // to tell CharBuffer is modified if ((s=next(tokenPat))!=null) { @@ -199,6 +172,7 @@ } } + @Override public boolean hasRemaining() { return cb.hasRemaining()||extendInput(); } @@ -226,35 +200,8 @@ return false; } - protected PropertyListScanner<Node> pushScanner(String exp) { - // Save current matcher for nested parsing - return new PropertyListScanner<Node>(this,dict,nullToken).set(exp); - } - - protected PropertyListScanner<Node> pushScannerFile(String newfile) throws FileNotFoundException { - // Save current matcher for nested file - return new PropertyListScanner<Node>(this,dict,nullToken).setFile(newfile); - } - - public PropertyListScanner<Node> pushScannerFile(InputStream newfile,String prompt) { - return new PropertyListScanner<Node>(this,dict,nullToken).setFile(newfile,prompt); - } - - protected PropertyListScanner<Node> popScanner() { - return prev; - } - - private PropertyListScanner<Node> findFileName() { - for(PropertyListScanner<Node> s = this;s!=null ; s = s.prev) { - if (s.filename!=null) return s; - } - return null; - } - /* - * Read From String - */ - - public PropertyListScanner<Node> set(String exp) { + @Override + public PLScanner<Node> set(String exp) { cb = CharBuffer.wrap(exp); scan = tokenPat.matcher(cb); filename = null; file = null; @@ -266,14 +213,16 @@ * Read From File * We cannot read symbol bigger than Buffersize */ - public PropertyListScanner<Node> setFile(String file) throws FileNotFoundException { + @Override + public PLScanner<Node> setFile(String file) throws FileNotFoundException { this.filename = file; nextToken = nullToken; set(new FileReader(file)); return this; } - public PropertyListScanner<Node> set(InputStreamReader file) { + @Override + public PLScanner<Node> set(InputStreamReader file) { this.file = file; cb = CharBuffer.allocate(BufferSize); try { @@ -293,52 +242,20 @@ return this; } - public void error(String err) { - PropertyListScanner<Node> s = findFileName(); - if (s!=null) { - System.err.print(s.filename+":"+s.lineno+": "); - } - System.err.println("error: "+err); - } - /* - * Iterator for Test Routing - * for(Token<Node> t: scanner.scanToken(FileReader(file)) { ... } - */ - - public Iterable<Token<Node>> scanToken(String exp) { - set(exp); - return iterator(); - } - - public Iterable<Token<Node>> scanToken(FileReader file) { - set(file); - return iterator(); + @Override + public PLScanner<Node> pushScannerFile(InputStream newfile, String prompt) { + return new PropertyListScanner<Node>(this,dict,nullToken).setFile(newfile,prompt); } - private Iterable<Token<Node>> iterator() { - return new Iterable<Token<Node>>() { - public Iterator<Token<Node>> iterator() { - return new Iterator<Token<Node>>() { - public boolean hasNext() { - return hasRemaining(); - } - public Token<Node> next() { - return nextToken(); - } - public void remove() { - } - }; - } - }; + @Override + public PLScanner<Node> pushScanner(String exp) { + return new PropertyListScanner<Node>(this,dict,nullToken).set(exp); } - private PropertyListScanner<Node> setFile(InputStream newfile,String prompt) { - this.filename = newfile.toString(); - nextToken = nullToken; - this.prompt = prompt; - set(new InputStreamReader(newfile)); - return this; + @Override + public PLScanner<Node> pushScannerFile(String newfile) + throws FileNotFoundException { + return new PropertyListScanner<Node>(this,dict,nullToken).setFile(newfile); } - }
--- a/src/plparser/TestScanner.java Mon Aug 30 12:35:23 2010 +0900 +++ b/src/plparser/TestScanner.java Wed Sep 01 15:23:37 2010 +0900 @@ -6,7 +6,7 @@ public class TestScanner { - public static PropertyListScanner<Property> scan; + public static PLScanner<Property> scan; public static void main(String arg[]) { initScanner();