Mercurial > hg > Members > kono > PLparser

changeset 10:0d74081c1309

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Char loop tokenizer worked.
author one
date Thu, 02 Sep 2010 10:52:44 +0900
parents 29e309b2f624
children 79d492bce828
files src/plparser/PropertyListCharTokenizer.java src/plparser/PropertyListParser.java src/plparser/PropertyListScanner.java
diffstat 3 files changed, 65 insertions(+), 51 deletions(-) [+]
line wrap: on
line diff
--- a/src/plparser/PropertyListCharTokenizer.java	Thu Sep 02 10:00:23 2010 +0900
+++ b/src/plparser/PropertyListCharTokenizer.java	Thu Sep 02 10:52:44 2010 +0900
@@ -5,7 +5,7 @@
 import java.nio.CharBuffer;
 
 public class PropertyListCharTokenizer<T> extends PropertyListScanner<T>
-		implements PLScanner<T> {
+implements PLScanner<T> {
 
 	public PropertyListCharTokenizer(Dictionary<T> dict) {
 		super(dict);
@@ -15,72 +15,81 @@
 			PLScanner<T>s,
 			Dictionary<T> dict, Token<T> nullToken) {
 		super(dict);
+		this.prev = s;
 		this.nullToken = nullToken;
-		this.prev = null;
 	}
 
 	public char ch;
-	
+
 	@Override
 	public Token<T> nextToken() {
 		nextToken = nullToken;
 		if (cb==null) return nextToken;
-		if (!hasRemaining()) return nextToken;
-		while(Character.isSpaceChar(ch)) {
+		while(true) {
 			if (!hasRemaining()) return nextToken;
-			ch = nextChar(); 
-		}
-		CharBuffer w = CharBuffer.allocate(BufferSize);
-		if (Character.isJavaIdentifierStart(ch)) {
-			w.put(ch);
-			while(hasRemaining()&&Character.isJavaIdentifierPart((ch=nextChar()))) {
-				w.put(ch);
-			}
-			return lookupDict(w);
-		} else if (Character.isDigit(ch)||ch=='-'||ch=='+') {
-			w.put(ch);
-			while(hasRemaining()&&Character.isDigit((ch=nextChar()))) {
-				w.put(ch);
+			while(Character.isSpaceChar(ch)) {
+				if (!hasRemaining()) return nextToken;
+				ch = nextChar(); 
 			}
-			return nextToken = new Token<T>(w.toString(),TokenID.NUMBER);
-		} else if (ch=='/') {
-			w.put(ch); 	
-			if (!hasRemaining()) return new Token<T>(w.toString(),TokenID.Any);
-			ch = nextChar();
-			if (ch=='/') {
-				while(hasRemaining() && (ch=nextChar())!='\n');
-				if (!hasRemaining())return nullToken;
+			CharBuffer w = CharBuffer.allocate(BufferSize);
+			if (Character.isJavaIdentifierStart(ch)) {
+				w.put(ch);
+				while(hasRemaining()&&Character.isJavaIdentifierPart((ch=nextChar()))) {
+					w.put(ch);
+				}
+				return lookupDict(w);
+			} else if (Character.isDigit(ch)) { // should handle more complex case
+				w.put(ch);
+				while(hasRemaining()&&Character.isDigit((ch=nextChar()))) {
+					w.put(ch);
+				}
+				return nextToken = new Token<T>(w.flip().toString(),TokenID.NUMBER);
+			} 
+			switch(ch) {
+			case '/':
+				w.put(ch); 	
+				if (!hasRemaining()) return new Token<T>(w.flip().toString(),TokenID.Any);
 				ch = nextChar();
-				return nextToken();
-			}
-			if (ch=='*') {
-				while(hasRemaining() && !((ch=nextChar())=='*'&&(ch=nextChar())=='/'));
-				if (!hasRemaining())return nullToken;
+				if (ch=='/') {
+					while(hasRemaining() && (ch=nextChar())!='\n');
+					if (!hasRemaining())return nullToken;
+					ch = nextChar();
+					continue;
+				}
+				if (ch=='*') {
+					while(hasRemaining() && !((ch=nextChar())=='*'&&(ch=nextChar())=='/'));
+					if (!hasRemaining())return nullToken;
+					ch = nextChar();
+					continue;
+				}
+				return new Token<T>(w.flip().toString(),TokenID.Any);
+			case '\'':   // should handle '\'' case
+			case '"':				
+				char d = ch;
+				while(hasRemaining() && (ch=nextChar())!=d) w.put(ch);
+				if (!hasRemaining())return nullToken; // non terminate string
 				ch = nextChar();
-				return nextToken();
+				Token<T> t = lookupDict(w);
+				if (t.type!=TokenID.VARIABLE) {
+					t = new Token<T>(t.name,TokenID.VARIABLE);
+				}
+				return nextToken = t;
+			case '{': case '}': case '(': case ')': case '=': case ',': case ';': 
+				w.put(ch); 	
+				nextToken = lookupDict(w);
+				if (!hasRemaining())return nextToken; 
+				ch = nextChar();
+				return nextToken;
+			default:
+				ch = nextChar();
+				continue;
 			}
-			return new Token<T>(w.toString(),TokenID.Any);
-		} else if (ch=='\'') {
-			while(hasRemaining() && (ch=nextChar())!='\'') w.put(ch);
-			if (!hasRemaining())return nullToken; // non terminate string
-			ch = nextChar();
-			return lookupDict(w);
-		} else if (ch=='"') {
-			while(hasRemaining() && (ch=nextChar())!='"') w.put(ch);
-			if (!hasRemaining())return nullToken; // non terminate string
-			ch = nextChar();
-			return lookupDict(w);
-		} else {
-			nextToken = lookupDict(w);
-			if (!hasRemaining())return nextToken; 
-			ch = nextChar();
-			return nextToken;
 		}
 	}
 
 	private Token<T> lookupDict(CharBuffer w) {
 		Token<T> t;
-		String s = w.toString();
+		String s = w.flip().toString();
 		if ((t = dict.get(s))==null) {
 			dict.put(s, t = new Token<T>(s,TokenID.Any));
 		}
@@ -90,6 +99,7 @@
 	private char nextChar() {
 		if (!cb.hasRemaining()) extendInput();
 		char ch = cb.get();
+		if (ch=='\n') lineno++;
 		return ch;
 	}
 
@@ -106,7 +116,7 @@
 
 	@Override
 	public PLScanner<T> pushScannerFile(String newfile)
-			throws FileNotFoundException {
+	throws FileNotFoundException {
 		return new PropertyListCharTokenizer<T>(this,dict,nullToken).setFile(newfile);
 	}
 }
--- a/src/plparser/PropertyListParser.java	Thu Sep 02 10:00:23 2010 +0900
+++ b/src/plparser/PropertyListParser.java	Thu Sep 02 10:52:44 2010 +0900
@@ -44,7 +44,8 @@
 		dict = new Dictionary<Node>();
 	//	scope = new PropertyListScope<Node>(null,dict);
 		initReservedWord();
-		scanner = new PropertyListScanner<Node>(dict);
+		// scanner = new PropertyListScanner<Node>(dict);
+		scanner = new PropertyListCharTokenizer<Node>(dict);
 	}
 
 	public Node parse() {
--- a/src/plparser/PropertyListScanner.java	Thu Sep 02 10:00:23 2010 +0900
+++ b/src/plparser/PropertyListScanner.java	Thu Sep 02 10:52:44 2010 +0900
@@ -105,6 +105,9 @@
 				if ((t = dict.get(s))==null) {
 					dict.put(s, t = new Token<Node>(s,TokenID.VARIABLE));
 				}
+				if (t.type!=TokenID.VARIABLE) {
+					t = new Token<Node>(s,TokenID.VARIABLE);
+				}
 				return nextToken = t;
 			} else if ((s=next(stringPat))!=null||(s=next(stringPat1))!=null||(s=next(namePat))!=null) {
 				Token<Node> t;