View Javadoc
1 package net.sourceforge.pmd.cpd; 2 3 import net.sourceforge.pmd.PMD; 4 import net.sourceforge.pmd.ast.JavaCharStream; 5 import net.sourceforge.pmd.ast.JavaParserTokenManager; 6 import net.sourceforge.pmd.ast.Token; 7 8 import java.io.IOException; 9 import java.io.LineNumberReader; 10 import java.io.Reader; 11 import java.io.StringReader; 12 import java.util.ArrayList; 13 import java.util.List; 14 15 public class JavaTokenizer implements Tokenizer { 16 17 public void tokenize(SourceCode tokens, Tokens tokenEntries, Reader input) throws IOException { 18 // first get a snapshot of the code 19 List lines = new ArrayList(); 20 StringBuffer sb = new StringBuffer(); 21 LineNumberReader r = new LineNumberReader(input); 22 String currentLine; 23 while ((currentLine = r.readLine()) != null) { 24 lines.add(currentLine); 25 sb.append(currentLine); 26 sb.append(PMD.EOL); 27 } 28 tokens.setCode(lines); 29 30 // now tokenize it 31 /* 32 I'm doing a sort of State pattern thing here where 33 this goes into "discarding" mode when it hits an import or package 34 keyword and goes back into "accumulate mode when it hits a semicolon. 35 This could probably be turned into some objects. 36 */ 37 JavaCharStream javaStream = new JavaCharStream(new StringReader(sb.toString())); 38 JavaParserTokenManager tokenMgr = new JavaParserTokenManager(javaStream); 39 Token currToken = tokenMgr.getNextToken(); 40 boolean discarding = false; 41 int count = 0; 42 while (currToken.image != "") { 43 if (currToken.image.equals("import") || currToken.image.equals("package")) { 44 discarding = true; 45 currToken = tokenMgr.getNextToken(); 46 continue; 47 } 48 49 if (discarding && currToken.image.equals(";")) { 50 discarding = false; 51 } 52 53 if (discarding) { 54 currToken = tokenMgr.getNextToken(); 55 continue; 56 } 57 58 if (!currToken.image.equals(";")) { 59 count++; 60 tokenEntries.add(new TokenEntry(currToken.image, count, tokens.getFileName(), currToken.beginLine)); 61 } 62 63 currToken = tokenMgr.getNextToken(); 64 } 65 tokenEntries.add(TokenEntry.EOF); 66 } 67 }

This page was automatically generated by Maven