View Javadoc

1   /***
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    */
4   package net.sourceforge.pmd.cpd;
5   
6   import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
7   import net.sourceforge.pmd.cpd.cppast.Token;
8   import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
9   import net.sourceforge.pmd.cpd.cppast.SimpleCharStream;
10  
11  import java.io.StringReader;
12  import java.util.ArrayList;
13  import java.util.List;
14  
15  public class CPPTokenizer implements Tokenizer {
16      protected String EOL = System.getProperty("line.separator", "\n");
17  
18      public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
19          StringBuffer sb = sourceCode.getCodeBuffer();
20          try {
21              CPPParserTokenManager tokenManager = new CPPParserTokenManager(new SimpleCharStream(new StringReader(sb.toString())));
22              Token currToken = tokenManager.getNextToken();
23              while (currToken.image.length() > 0) {
24                  tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine));
25                  currToken = tokenManager.getNextToken();
26              }
27              tokenEntries.add(TokenEntry.getEOF());
28              System.out.println("Added " + sourceCode.getFileName());
29          } catch (TokenMgrError err) {
30              err.printStackTrace();
31              System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error");
32              List emptyCode = new ArrayList();
33              emptyCode.add("");
34              tokenEntries.add(TokenEntry.getEOF());
35          }
36      }
37  }