View Javadoc

1   /***
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3   */
4   package net.sourceforge.pmd.cpd;
5   
6   import net.sourceforge.pmd.cpd.cppast.CPPParser;
7   import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
8   import net.sourceforge.pmd.cpd.cppast.Token;
9   import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
10  
11  import java.io.StringReader;
12  import java.util.ArrayList;
13  import java.util.List;
14  
15  public class CPPTokenizer implements Tokenizer {
16      protected String EOL = System.getProperty("line.separator", "\n");
17  
18      private static boolean initted;
19  
20      public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
21          StringBuffer sb = sourceCode.getCodeBuffer();
22          try {
23              // now tokenize it
24              if (!initted) {
25                  new CPPParser(new StringReader(sb.toString()));
26                  initted = true;
27              }
28              CPPParser.ReInit(new StringReader(sb.toString()));
29              Token currToken = CPPParserTokenManager.getNextToken();
30              int count = 0;
31              while (currToken.image != "") {
32                  count++;
33                  tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine));
34                  currToken = CPPParserTokenManager.getNextToken();
35              }
36              tokenEntries.add(TokenEntry.getEOF());
37              System.out.println("Added " + sourceCode.getFileName());
38          } catch (TokenMgrError err) {
39              System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error");
40              List emptyCode = new ArrayList();
41              emptyCode.add("");
42              tokenEntries.add(TokenEntry.getEOF());
43          }
44      }
45  }