1 /***
2 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3 */
4 package net.sourceforge.pmd.cpd;
5
6 import net.sourceforge.pmd.TargetJDK1_4;
7 import net.sourceforge.pmd.ast.JavaParserConstants;
8 import net.sourceforge.pmd.ast.JavaParserTokenManager;
9 import net.sourceforge.pmd.ast.Token;
10
11 import java.io.StringReader;
12 import java.util.Properties;
13
14 public class JavaTokenizer implements Tokenizer {
15
16 public static final String IGNORE_LITERALS = "ignore_literals";
17 public static final String IGNORE_IDENTIFIERS = "ignore_identifiers";
18
19 private boolean ignoreLiterals;
20 private boolean ignoreIdentifiers;
21
22 public void setProperties(Properties properties) {
23 ignoreLiterals = Boolean.valueOf(properties.getProperty(IGNORE_LITERALS, "false")).booleanValue();
24 ignoreIdentifiers = Boolean.valueOf(properties.getProperty(IGNORE_IDENTIFIERS, "false")).booleanValue();
25 }
26
27 public void tokenize(SourceCode tokens, Tokens tokenEntries) {
28 StringBuffer buffer = tokens.getCodeBuffer();
29
30
31
32
33
34
35
36
37
38 JavaParserTokenManager tokenMgr = new TargetJDK1_4().createJavaParserTokenManager(new StringReader(buffer.toString()));
39 Token currentToken = tokenMgr.getNextToken();
40 boolean inDiscardingState = false;
41 while (currentToken.image.length() > 0) {
42 if (currentToken.kind == JavaParserConstants.IMPORT || currentToken.kind == JavaParserConstants.PACKAGE) {
43 inDiscardingState = true;
44 currentToken = tokenMgr.getNextToken();
45 continue;
46 }
47
48 if (inDiscardingState && currentToken.kind == JavaParserConstants.SEMICOLON) {
49 inDiscardingState = false;
50 }
51
52 if (inDiscardingState) {
53 currentToken = tokenMgr.getNextToken();
54 continue;
55 }
56
57 if (currentToken.kind != JavaParserConstants.SEMICOLON) {
58 String image = currentToken.image;
59 if (ignoreLiterals && (currentToken.kind == JavaParserConstants.STRING_LITERAL || currentToken.kind == JavaParserConstants.CHARACTER_LITERAL
60 || currentToken.kind == JavaParserConstants.DECIMAL_LITERAL || currentToken.kind == JavaParserConstants.FLOATING_POINT_LITERAL)) {
61 image = String.valueOf(currentToken.kind);
62 }
63 if (ignoreIdentifiers && currentToken.kind == JavaParserConstants.IDENTIFIER) {
64 image = String.valueOf(currentToken.kind);
65 }
66 tokenEntries.add(new TokenEntry(image, tokens.getFileName(), currentToken.beginLine));
67 }
68
69 currentToken = tokenMgr.getNextToken();
70 }
71 tokenEntries.add(TokenEntry.getEOF());
72 }
73
74 public void setIgnoreLiterals(boolean ignore) {
75 this.ignoreLiterals = ignore;
76 }
77
78 public void setIgnoreIdentifiers(boolean ignore) {
79 this.ignoreIdentifiers = ignore;
80 }
81 }