PythonCpdAnalyzer { public void pushCpdTokens(InputFile inputFile, PythonVisitorContext visitorContext) { Tree root = visitorContext.rootTree(); if (root != null) { NewCpdTokens cpdTokens = context.newCpdTokens().onFile(inputFile); List<Token> tokens = TreeUtils.tokens(root); for (int i = 0; i < tokens.size(); i++) { Token token = tokens.get(i); TokenType currentTokenType = token.type(); TokenType nextTokenType = i + 1 < tokens.size() ? tokens.get(i + 1).type() : GenericTokenType.EOF; if (isNewLineWithIndentationChange(currentTokenType, nextTokenType) || !isIgnoredType(currentTokenType)) { TokenLocation location = new TokenLocation(token); cpdTokens.addToken(location.startLine(), location.startLineOffset(), location.endLine(), location.endLineOffset(), token.value()); } } cpdTokens.save(); } } PythonCpdAnalyzer(SensorContext context); void pushCpdTokens(InputFile inputFile, PythonVisitorContext visitorContext);  }
@Test public void code_chunks_2() { DefaultInputFile inputFile = inputFile("code_chunks_2.py"); PythonVisitorContext visitorContext = TestPythonVisitorRunner.createContext(inputFile.path().toFile()); cpdAnalyzer.pushCpdTokens(inputFile, visitorContext); List<TokensLine> lines = context.cpdTokens("moduleKey:code_chunks_2.py"); assertThat(lines).isNotNull().hasSize(29); TokensLine line1 = lines.get(0); assertThat(line1.getStartLine()).isEqualTo(2); assertThat(line1.getEndLine()).isEqualTo(2); assertThat(line1.getStartUnit()).isEqualTo(1); assertThat(line1.getEndUnit()).isEqualTo(1); List<String> values = lines.stream().map(TokensLine::getValue).collect(Collectors.toList()); assertThat(values).containsExactly( "00000", "1111L", "0x10000L", "0X1111", "0b1111L", "0o12345", "u\"lala\"", "U\"lala\"", "r\"lala\"", "R\"lala\"", "print", "a=[1,", "2,", "]", "deffoo():pass", "classbar(object):pass", "deffoo2(x,y,z,):", "pass\n", "defbar(*baz):", "foo(3,4,5)\n", "defbar2(**baz):", "yield;\n", "items=[]", "(itemforiteminitems)", "[itemforiteminitems]", "ifitemisnotNone:", "pass\n", "ifitemnotinitems:", "pass\n"); }
@Test public void dedent_with_cpd() { DefaultInputFile inputFile = inputFile("cpd_dedent.py"); PythonVisitorContext visitorContext = TestPythonVisitorRunner.createContext(inputFile.path().toFile()); cpdAnalyzer.pushCpdTokens(inputFile, visitorContext); List<TokensLine> tokensLines = context.cpdTokens("moduleKey:cpd_dedent.py"); assertThat(tokensLines).isNotNull(); assertThat(tokensLines.size() % 2).isEqualTo(0); int mid = tokensLines.size() / 2; for (int i = 0; i < mid; i++) { TokensLine tokensLine = tokensLines.get(i); TokensLine dup = tokensLines.get(mid + i); assertThat(tokensLine.getStartLine() + 5).isEqualTo(dup.getStartLine()); if (dup.getStartLine() == 9) { assertThat(tokensLine.getValue()).isNotEqualTo(dup.getValue()); } else { assertThat(tokensLine.getValue()).isEqualTo(dup.getValue()); } } }