1 package net.sourceforge.pmd.cpd;
2
3 import net.sourceforge.pmd.cpd.cppast.CPPParser;
4 import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
5 import net.sourceforge.pmd.cpd.cppast.Token;
6 import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
7
8 import java.io.IOException;
9 import java.io.LineNumberReader;
10 import java.io.Reader;
11 import java.io.StringReader;
12 import java.util.ArrayList;
13 import java.util.List;
14
15 public class CPPTokenizer implements Tokenizer {
16 protected String EOL = System.getProperty("line.separator", "\n");
17
18 private static boolean initted;
19
20 public void tokenize(SourceCode sourceCode, Tokens tokenEntries, Reader input) throws IOException {
21 try {
22 // first get a snapshot of the code
23 List lines = new ArrayList();
24 StringBuffer sb = new StringBuffer();
25 LineNumberReader r = new LineNumberReader(input);
26 String currentLine;
27 while ((currentLine = r.readLine()) != null) {
28 lines.add(currentLine);
29 sb.append(currentLine);
30 sb.append(EOL);
31 }
32 sourceCode.setCode(lines);
33
34 // now tokenize it
35 if (!initted) {
36 new CPPParser(new StringReader(sb.toString()));
37 initted = true;
38 }
39 CPPParser.ReInit(new StringReader(sb.toString()));
40 Token currToken = CPPParserTokenManager.getNextToken();
41 int count = 0;
42 while (currToken.image != "") {
43 count++;
44 tokenEntries.add(new TokenEntry(currToken.image, count, sourceCode.getFileName(), currToken.beginLine));
45 currToken = CPPParserTokenManager.getNextToken();
46 }
47 tokenEntries.add(TokenEntry.EOF);
48 System.out.println("Added " + sourceCode.getFileName());
49 } catch (TokenMgrError err) {
50 System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error");
51 List emptyCode = new ArrayList();
52 emptyCode.add("");
53 sourceCode.setCode(emptyCode);
54 tokenEntries.add(TokenEntry.EOF);
55 }
56 }
57 }
This page was automatically generated by Maven