1 /***
2 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3 */
4 package net.sourceforge.pmd.cpd;
5
6 import net.sourceforge.pmd.cpd.cppast.CPPParser;
7 import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
8 import net.sourceforge.pmd.cpd.cppast.Token;
9 import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
10
11 import java.io.StringReader;
12 import java.util.ArrayList;
13 import java.util.List;
14
15 public class CPPTokenizer implements Tokenizer {
16 protected String EOL = System.getProperty("line.separator", "\n");
17
18 private static boolean initted;
19
20 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
21 StringBuffer sb = sourceCode.getCodeBuffer();
22 try {
23
24 if (!initted) {
25 new CPPParser(new StringReader(sb.toString()));
26 initted = true;
27 }
28 CPPParser.ReInit(new StringReader(sb.toString()));
29 Token currToken = CPPParserTokenManager.getNextToken();
30 int count = 0;
31 while (currToken.image != "") {
32 count++;
33 tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine));
34 currToken = CPPParserTokenManager.getNextToken();
35 }
36 tokenEntries.add(TokenEntry.getEOF());
37 System.out.println("Added " + sourceCode.getFileName());
38 } catch (TokenMgrError err) {
39 System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error");
40 List emptyCode = new ArrayList();
41 emptyCode.add("");
42 tokenEntries.add(TokenEntry.getEOF());
43 }
44 }
45 }