1 /*** 2 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html 3 */ 4 package net.sourceforge.pmd.cpd; 5 6 import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager; 7 import net.sourceforge.pmd.cpd.cppast.Token; 8 import net.sourceforge.pmd.cpd.cppast.TokenMgrError; 9 import net.sourceforge.pmd.cpd.cppast.SimpleCharStream; 10 11 import java.io.StringReader; 12 import java.util.ArrayList; 13 import java.util.List; 14 15 public class CPPTokenizer implements Tokenizer { 16 protected String EOL = System.getProperty("line.separator", "\n"); 17 18 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { 19 StringBuffer sb = sourceCode.getCodeBuffer(); 20 try { 21 CPPParserTokenManager tokenManager = new CPPParserTokenManager(new SimpleCharStream(new StringReader(sb.toString()))); 22 Token currToken = tokenManager.getNextToken(); 23 while (currToken.image.length() > 0) { 24 tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine)); 25 currToken = tokenManager.getNextToken(); 26 } 27 tokenEntries.add(TokenEntry.getEOF()); 28 System.out.println("Added " + sourceCode.getFileName()); 29 } catch (TokenMgrError err) { 30 err.printStackTrace(); 31 System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error"); 32 List emptyCode = new ArrayList(); 33 emptyCode.add(""); 34 tokenEntries.add(TokenEntry.getEOF()); 35 } 36 } 37 }