1 /**
2 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3 */
4 package net.sourceforge.pmd.cpd;
5
6 import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
7 import net.sourceforge.pmd.cpd.cppast.SimpleCharStream;
8 import net.sourceforge.pmd.cpd.cppast.Token;
9 import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
10
11 import java.io.StringReader;
12
13 public class CPPTokenizer implements Tokenizer {
14 private static SimpleCharStream charStream;
15
16 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
17 StringBuffer sb = sourceCode.getCodeBuffer();
18 try {
19 if (charStream == null) {
20 charStream = new SimpleCharStream(new StringReader(sb.toString()));
21 } else {
22 charStream.ReInit(new StringReader(sb.toString()));
23 }
24 CPPParserTokenManager.ReInit(charStream);
25 CPPParserTokenManager.setFileName(sourceCode.getFileName());
26 Token currToken = CPPParserTokenManager.getNextToken();
27 while (currToken.image.length() > 0) {
28 tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine));
29 currToken = CPPParserTokenManager.getNextToken();
30 }
31 tokenEntries.add(TokenEntry.getEOF());
32 System.err.println("Added " + sourceCode.getFileName());
33 } catch (TokenMgrError err) {
34 err.printStackTrace();
35 System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
36 tokenEntries.add(TokenEntry.getEOF());
37 }
38 }
39 }