PythonArchiveAnalyzer implements FileAnalyzer { @Override public Map<ConstructId, Construct> getConstructs() throws FileAnalysisException { if (this.constructs.isEmpty()) { final List<String> inits = new ArrayList<String>(); boolean pre_processed = false; try (InputStream is = this.getArchiveInputStream()) { if (is instanceof ZipInputStream) { ZipEntry en = null; while ((en = ((ZipInputStream) is).getNextEntry()) != null) { if (!en.isDirectory()) { if (this.isPackage(en.getName())) { inits.add(en.getName()); } } } } else if (is instanceof TarArchiveInputStream) { ArchiveEntry en = null; while ((en = ((TarArchiveInputStream) is).getNextEntry()) != null) { if (!en.isDirectory()) { if (this.isPackage(en.getName())) { inits.add(en.getName()); } } } } pre_processed = true; } catch (IOException e) { log.error( "IOException analyzing Python archive [" + this.archive.getAbsolutePath() + "]", e); } try (InputStream is = this.getArchiveInputStream()) { if (is instanceof ZipInputStream && pre_processed) { ZipEntry en = null; while ((en = ((ZipInputStream) is).getNextEntry()) != null) { if (!en.isDirectory()) { if (en.getName().endsWith(".whl") || en.getName().endsWith(".egg") || en.getName().endsWith(".gz")) { final FileAnalyzer fa = DirAnalyzer.createAnalyzerForArchiveEntry(is, en.getName()); if (fa != null) this.nestedAnalyzers.add(fa); } else if (en.getName().endsWith(".py")) { final List<String> package_name = this.getPackageName(en.getName(), inits); PythonId pack = null; if (!package_name.isEmpty()) pack = new PythonId(null, PythonId.Type.PACKAGE, StringUtil.join(package_name, ".")); final String module_name = this.getModuleName(en.getName()); final PythonId module = new PythonId(pack, PythonId.Type.MODULE, module_name); final byte[] py_bytes = FileUtil.readInputStream(is); final FileAnalyzer fa = PythonFileAnalyzer.createAnalyzer(new ByteArrayInputStream(py_bytes)); if (fa instanceof Python3FileAnalyzer) { ((Python3FileAnalyzer) fa).setContext(module, pack); this.constructs.putAll( ((Python3FileAnalyzer) fa).getConstructs(new ByteArrayInputStream(py_bytes))); } else if (fa instanceof Python335FileAnalyzer) { ((Python335FileAnalyzer) fa).setContext(module, pack); this.constructs.putAll( ((Python335FileAnalyzer) fa) .getConstructs(new ByteArrayInputStream(py_bytes))); } } } } } else if (is instanceof TarArchiveInputStream) { ArchiveEntry en = null; while ((en = ((TarArchiveInputStream) is).getNextEntry()) != null) { if (!en.isDirectory()) { if (en.getName().endsWith(".whl") || en.getName().endsWith(".egg") || en.getName().endsWith(".gz")) { final FileAnalyzer fa = DirAnalyzer.createAnalyzerForArchiveEntry(is, en.getName()); if (fa != null) this.nestedAnalyzers.add(fa); } else if (en.getName().endsWith(".py")) { final List<String> package_name = this.getPackageName(en.getName(), inits); PythonId pack = null; if (!package_name.isEmpty()) pack = new PythonId(null, PythonId.Type.PACKAGE, StringUtil.join(package_name, ".")); final String module_name = this.getModuleName(en.getName()); final PythonId module = new PythonId(pack, PythonId.Type.MODULE, module_name); final byte[] py_bytes = FileUtil.readInputStream(is); final FileAnalyzer fa = PythonFileAnalyzer.createAnalyzer(new ByteArrayInputStream(py_bytes)); if (fa instanceof Python3FileAnalyzer) { ((Python3FileAnalyzer) fa).setContext(module, pack); this.constructs.putAll( ((Python3FileAnalyzer) fa).getConstructs(new ByteArrayInputStream(py_bytes))); } else if (fa instanceof Python335FileAnalyzer) { ((Python335FileAnalyzer) fa).setContext(module, pack); this.constructs.putAll( ((Python335FileAnalyzer) fa) .getConstructs(new ByteArrayInputStream(py_bytes))); } } } } } } catch (IOException e) { log.error( "IOException analyzing Python archive [" + this.archive.getAbsolutePath() + "]", e); } } return constructs; }  @Override String[] getSupportedFileExtensions(); @Override boolean canAnalyze(File _file); @Override void analyze(final File _file); Path getArchivePath(); String getDigest(); @Override Map<ConstructId, Construct> getConstructs(); Library getLibrary(); @Override boolean containsConstruct(ConstructId _id); @Override Construct getConstruct(ConstructId _id); @Override boolean hasChilds(); @Override Set<FileAnalyzer> getChilds(boolean _recursive); List<org.eclipse.steady.shared.json.model.ConstructId> getSharedConstructs();  }
@Test public void testGunicorn() throws FileAnalysisException { final FileAnalyzer f3 = FileAnalyzerFactory.buildFileAnalyzer( new File("src/test/resources/gunicorn-19.7.1-py2.py3-none-any.whl")); final Map<ConstructId, Construct> c3 = f3.getConstructs(); final PythonId pack = new PythonId(null, PythonId.Type.PACKAGE, "gunicorn.workers"); PythonId modu = new PythonId(pack, PythonId.Type.MODULE, "_gaiohttp"); final PythonId clas = new PythonId(modu, PythonId.Type.CLASS, "AiohttpWorker(base.Worker)"); final PythonId meth = new PythonId(clas, PythonId.Type.METHOD, "run(self)"); assertTrue(c3.containsKey(pack)); assertTrue(c3.containsKey(modu)); assertTrue(c3.containsKey(clas)); assertTrue(c3.containsKey(meth)); modu = new PythonId(pack, PythonId.Type.MODULE, "geventlet"); assertTrue(c3.containsKey(modu)); modu = new PythonId(pack, PythonId.Type.MODULE, "ggevent"); assertTrue(c3.containsKey(modu)); }
@Test public void testPythonArchiveWithAsyncAnalyzer() { try { final FileAnalyzer f4 = FileAnalyzerFactory.buildFileAnalyzer( new File("src/test/resources/gevent-1.2.2-cp35-cp35m-win_amd64.whl")); assertTrue(f4 instanceof PythonArchiveAnalyzer); final PythonArchiveAnalyzer paa = (PythonArchiveAnalyzer) f4; final Map<ConstructId, Construct> c4 = paa.getConstructs(); } catch (IllegalArgumentException e) { e.printStackTrace(); assertTrue(false); } catch (FileAnalysisException e) { e.printStackTrace(); assertTrue(false); } }
@Test public void testPythonArchiveFormats() throws FileAnalysisException { final FileAnalyzer f1 = FileAnalyzerFactory.buildFileAnalyzer( new File("src/test/resources/monotonic-1.3-py2.py3-none-any.whl")); final Map<ConstructId, Construct> c1 = f1.getConstructs(); assertEquals(7, c1.size()); final FileAnalyzer f2 = FileAnalyzerFactory.buildFileAnalyzer( new File("src/test/resources/setuptools_git-1.2-py3.6.egg")); final Map<ConstructId, Construct> c2 = f2.getConstructs(); assertEquals(58, c2.size()); final FileAnalyzer f4 = FileAnalyzerFactory.buildFileAnalyzer( new File("src/test/resources/itsdangerous-0.24.tar.gz")); final Map<ConstructId, Construct> c4 = f4.getConstructs(); assertEquals(128, c4.size()); }
@Test public void testFlaskOIDCConstructs() throws FileAnalysisException { final FileAnalyzer f1 = FileAnalyzerFactory.buildFileAnalyzer( new File("./src/test/resources/flask-oidc_e4ce5/flask_oidc/__init__.py")); final Map<ConstructId, Construct> c1 = f1.getConstructs(); final FileAnalyzer f2 = FileAnalyzerFactory.buildFileAnalyzer( new File("./src/test/resources/flask-oidc_f2ef8/flask_oidc/__init__.py")); final Map<ConstructId, Construct> c2 = f2.getConstructs(); assertEquals(c1.size(), c2.size()); }