View Javadoc
1 package net.sourceforge.pmd.cpd; 2 3 import net.sourceforge.pmd.cpd.cppast.CPPParser; 4 import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager; 5 import net.sourceforge.pmd.cpd.cppast.Token; 6 import net.sourceforge.pmd.cpd.cppast.TokenMgrError; 7 8 import java.io.IOException; 9 import java.io.LineNumberReader; 10 import java.io.Reader; 11 import java.io.StringReader; 12 import java.util.ArrayList; 13 import java.util.List; 14 15 public class CPPTokenizer implements Tokenizer { 16 protected String EOL = System.getProperty("line.separator", "\n"); 17 18 private static boolean initted; 19 20 public void tokenize(SourceCode sourceCode, Tokens tokenEntries, Reader input) throws IOException { 21 try { 22 // first get a snapshot of the code 23 List lines = new ArrayList(); 24 StringBuffer sb = new StringBuffer(); 25 LineNumberReader r = new LineNumberReader(input); 26 String currentLine; 27 while ((currentLine = r.readLine()) != null) { 28 lines.add(currentLine); 29 sb.append(currentLine); 30 sb.append(EOL); 31 } 32 sourceCode.setCode(lines); 33 34 // now tokenize it 35 if (!initted) { 36 new CPPParser(new StringReader(sb.toString())); 37 initted = true; 38 } 39 CPPParser.ReInit(new StringReader(sb.toString())); 40 Token currToken = CPPParserTokenManager.getNextToken(); 41 int count = 0; 42 while (currToken.image != "") { 43 count++; 44 tokenEntries.add(new TokenEntry(currToken.image, count, sourceCode.getFileName(), currToken.beginLine)); 45 currToken = CPPParserTokenManager.getNextToken(); 46 } 47 tokenEntries.add(TokenEntry.EOF); 48 System.out.println("Added " + sourceCode.getFileName()); 49 } catch (TokenMgrError err) { 50 System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error"); 51 List emptyCode = new ArrayList(); 52 emptyCode.add(""); 53 sourceCode.setCode(emptyCode); 54 tokenEntries.add(TokenEntry.EOF); 55 } 56 } 57 }

This page was automatically generated by Maven