import { IAqlSearchResult , IArtifact , IDetailsResponse , IGeneral , ISearchEntry } from 'jfrog-client-js' ; import PQueue from 'p-queue' ; import Set from 'typescript-collections/dist/lib/Set' ; import * as vscode from 'vscode' ; import { ConnectionUtils } from '../../connect/connectionUtils' ; import { BuildsScanCache , Type } from '../../scanCache/buildsScanCache' ; import { ScanCacheManager } from '../../scanCache/scanCacheManager' ; import { BuildsNode } from '../../treeDataProviders/dependenciesTree/ciNodes/buildsTree' ; import { CiTitleNode } from '../../treeDataProviders/dependenciesTree/ciNodes/ciTitleNode' ; import { DependenciesTreeNode } from '../../treeDataProviders/dependenciesTree/dependenciesTreeNode' ; import { TreesManager } from '../../treeDataProviders/treesManager' ; import { BuildGeneralInfo } from '../../types/buildGeneralinfo' ; import { Dependency } from '../../types/dependency' ; import { GeneralInfo } from '../../types/generalInfo' ; import { IIssueKey } from '../../types/issueKey' ; import { ILicenseKey } from '../../types/licenseKey' ; import { Severity } from '../../types/severity' ; import { Configuration } from '../configuration' ; import { Consts } from '../consts' ; import { ScanCancellationError } from '../scanUtils' ; import { Translators } from '../translators' ; import { BuildsUtils } from './buildsUtils' ; /**
 * Manage the filters of the components tree.
 */ export class CiManager { private static readonly DISPLAY_BUILDS_NUM : string = '100' ; private readonly buildsCache : BuildsScanCache ; private root : DependenciesTreeNode ; constructor ( private _treesManager , root ? ) { this . buildsCache = new BuildsScanCache ( Configuration . getBuildsPattern ( ) , this . _treesManager . logManager ) ; if ( ! root ) { this . root = new DependenciesTreeNode ( new GeneralInfo ( '' , '' , [ 'None' ] , '' , '' ) , vscode . TreeItemCollapsibleState . Expanded , undefined , '' ) ; } else { this . root = root ; } } public loadBuildTree ( timestamp , buildName , buildNumber , parent ) { let projectKey = Configuration . getProjectKey ( ) ; const build = this . buildsCache . loadBuildInfo ( timestamp , buildName , buildNumber , projectKey ) ; if ( ! build ) { throw new Error ( `Couldn't find build info object in cache for '${ buildName }/${ buildNumber }'.` ) ; } let bgi = BuildsUtils . createBuildGeneralInfo ( build , this . _treesManager . logManager ) ; const buildTree = new BuildsNode ( bgi ) ; parent . addChild ( buildTree ) ; const modulesTree = new CiTitleNode ( new GeneralInfo ( CiTitleNode . MODULES_NODE , '' , [ 'None' ] , '' , 'Build Modules' ) , vscode . TreeItemCollapsibleState . Expanded , parent ) ; if ( BuildsUtils . isArrayExistsAndNotEmpty ( build , CiTitleNode . MODULES_NODE ) ) { this . populateModulesDependencyTree ( build , modulesTree ) ; } let detailsResponse = this . buildsCache . loadScanResults ( timestamp , buildName , buildNumber , projectKey ) ; this . populateBuildDependencyTree ( detailsResponse , modulesTree ) ; modulesTree . issues = modulesTree . processTreeIssues ( ) ; } public populateModulesDependencyTree ( build , modulesRoot ) { for ( const module of build . modules ) { const artifactId = BuildsUtils . getArtifactIdFromCompId ( module . id ) ; const version = BuildsUtils . getVersionFromCompId ( module . id ) ; const moduleGeneralInfo = new GeneralInfo ( artifactId , version , [ 'None' ] , '' , module . type ) ; const moduleNode = new CiTitleNode ( moduleGeneralInfo , vscode . TreeItemCollapsibleState . Collapsed , undefined ) ; if ( BuildsUtils . isArrayExistsAndNotEmpty ( module , CiTitleNode . ARTIFACTS_NODE ) ) { const artifactsNode = BuildsUtils . createArtifactsNode ( ) ; moduleNode . addChild ( artifactsNode ) ; this . populateArtifacts ( artifactsNode , module ) ; } if ( BuildsUtils . isArrayExistsAndNotEmpty ( module , CiTitleNode . DEPENDENCIES_NODE ) ) { const dependenciesNode = BuildsUtils . createDependenciesNode ( ) ; moduleNode . addChild ( dependenciesNode ) ; this . populateDependencies ( dependenciesNode , module ) ; } if ( moduleNode . children . length > 0 ) { modulesRoot . addChild ( moduleNode ) ; } } } public populateArtifacts ( artifactsNode , module ) { for ( const artifact of module . artifacts ) { const artifactGeneralInfo = new GeneralInfo ( BuildsUtils . getArtifactIdFromCompId ( artifact . name ) , BuildsUtils . getVersionFromCompId ( artifact . name ) , [ 'None' ] , '' , artifact . type , artifact . sha1 , artifact . sha256 ) ; const artifactNode = new DependenciesTreeNode ( artifactGeneralInfo , vscode . TreeItemCollapsibleState . None , undefined , '' ) ; artifactsNode . addChild ( artifactNode ) ; } } public populateDependencies ( dependenciesNode , module ) { let directDependencies = new Set < Dependency > ( dependency => dependency . id ) ; let parentToChildren = new Map < string , Dependency [ ] > ( ) ; for ( const dependency of module . dependencies ) { let requestedBy = dependency . requestedBy ; if ( ! ( BuildsUtils . isArrayExistsAndNotEmpty ( module , CiTitleNode . DEPENDENCIES_NODE ) && requestedBy && ! ! requestedBy [ 0 ] . length ) ) { directDependencies . add ( dependency ) ; continue ; } for ( const parent of requestedBy ) { let directParent = parent [ 0 ] ; if ( ! requestedBy [ 0 ] [ 0 ] || requestedBy [ 0 ] [ 0 ] === module . id ) { directDependencies . add ( dependency ) ; } else { let children = parentToChildren . get ( directParent ) || [ ] ; children . push ( dependency ) ; parentToChildren . set ( directParent , children ) ; } } } for ( const directDependency of directDependencies . toArray ( ) ) { dependenciesNode . addChild ( this . populateTransitiveDependencies ( directDependency , parentToChildren ) ) ; } } private populateTransitiveDependencies ( dependency , parentToChildren ) { let dependencyGeneralInfo = new GeneralInfo ( BuildsUtils . getArtifactIdFromCompId ( dependency . id ) , BuildsUtils . getVersionFromCompId ( dependency . id ) , dependency . scopes || [ 'None' ] , '' , dependency . type , dependency . sha1 , dependency . sha256 ) ; let dependencies = parentToChildren . get ( dependency . id ) || [ ] ; let dependencyTree = new DependenciesTreeNode ( dependencyGeneralInfo , dependencies . length > 0 ? vscode . TreeItemCollapsibleState . Collapsed : vscode . TreeItemCollapsibleState . None , undefined , '' ) ; for ( const child of dependencies ) { dependencyTree . addChild ( this . populateTransitiveDependencies ( child , parentToChildren ) ) ; } return dependencyTree ; } public populateBuildDependencyTree ( response , modulesTree ) { if ( ! response ) { this . populateTreeWithUnknownIssues ( modulesTree ) ; return ; } let componentIssuesAndLicenses = new Map ( ) ; let sha1ToSha256 = new Map ( ) ; let sha1ToComponent = new Map ( ) ; let scanCacheManager = this . _treesManager . scanCacheManager ; let components = response . components ; for ( const component of components ) { let general = component . general ; sha1ToComponent . set ( general . sha1 , component ) ; sha1ToSha256 . set ( general . sha1 , general . sha256 ) ; for ( const parentSha256 of general . parent_sha256 ) { let issuesAndLicenses = componentIssuesAndLicenses . get ( parentSha256 ) ; if ( ! issuesAndLicenses ) { issuesAndLicenses = new IssuesAndLicensesPair ( ) ; componentIssuesAndLicenses . set ( parentSha256 , issuesAndLicenses ) ; } if ( component . issues . length > 0 ) { for ( const issue of component . issues ) { issuesAndLicenses . _issues . add ( new IssueIdWithSeverity ( issue . issue_id , Translators . toSeverity ( issue . severity ) ) ) ; scanCacheManager . storeIssue ( Translators . toCacheIssue ( issue ) ) ; } } if ( component . licenses . length > 0 ) { for ( const license of component . licenses ) { issuesAndLicenses . _licenses . add ( license . name ) ; scanCacheManager . storeLicense ( Translators . toCacheLicense ( license ) ) ; } } } } for ( const module of modulesTree . children ) { for ( const artifactsOrDep of module . children ) { const isArtifactNode = artifactsOrDep . generalInfo . artifactId === CiTitleNode . ARTIFACTS_NODE ; for ( const child of artifactsOrDep . children ) { this . populateComponents ( child , sha1ToComponent , sha1ToSha256 , componentIssuesAndLicenses , isArtifactNode ) ; } } } } public populateComponents ( node , sha1ToComponent , sha1ToSha256 , componentIssuesAndLicenses , isArtifact ) { for ( const child of node . children ) { this . populateComponents ( child , sha1ToComponent , sha1ToSha256 , componentIssuesAndLicenses , isArtifact ) ; } let sha1 = node . generalInfo . sha1 ; if ( ! sha1 ) { return ; } let artifact = sha1ToComponent . get ( sha1 ) ; if ( ! artifact ) { this . addUnknownLicenseToMissingNode ( node ) ; return ; } if ( artifact . issues . length > 0 ) { let topSeverity = Severity . Normal ; for ( const issue of artifact . issues ) { node . issues . add ( { issue_id : issue . issue_id } as IIssueKey ) ; let issueSeverity = Translators . toSeverity ( issue . severity ) ; if ( topSeverity < issueSeverity ) { topSeverity = issueSeverity ; } } node . topSeverity = topSeverity ; } if ( artifact . licenses . length > 0 ) { for ( const license of artifact . licenses ) { node . licenses . add ( { licenseName : license . name } as ILicenseKey ) ; } } if ( ! isArtifact ) { return ; } let sha256 = sha1ToSha256 . get ( sha1 ) ; if ( ! sha256 ) { return ; } let issuesAndLicenses = componentIssuesAndLicenses . get ( sha256 ) ; if ( issuesAndLicenses ) { let topSeverity = Severity . Normal ; issuesAndLicenses . _issues . forEach ( issue => { node . issues . add ( { issue_id : issue . _issueId } as IIssueKey ) ; let issueSeverity = issue . _severity ; if ( topSeverity < issueSeverity ) { topSeverity = issueSeverity ; } } ) ; node . topSeverity = topSeverity ; issuesAndLicenses . _licenses . forEach ( licenseName => { node . licenses . add ( { licenseName } as ILicenseKey ) ; } ) ; } } private addUnknownLicenseToMissingNode ( node ) { node . licenses . add ( { licenseName : Consts . UNKNOWN_LICENSE } as ILicenseKey ) ; } public populateTreeWithUnknownIssues ( modulesTree ) { for ( const node of modulesTree . children ) { this . populateTreeWithUnknownIssues ( node ) ; } modulesTree . issues . add ( { issue_id : Consts . MISSING_COMPONENT } as IIssueKey ) ; this . addUnknownLicenseToMissingNode ( modulesTree ) ; } public async refreshBuilds ( progress , checkCanceled ) { const pattern = Configuration . getBuildsPattern ( ) ; try { await this . buildCiTree ( pattern , progress , checkCanceled ) ; } catch ( error ) { vscode . window . showErrorMessage ( 'Could not build CI tree.' , < vscode . MessageOptions > { modal : false } ) ; this . _treesManager . logManager . logMessage ( ( < any > error ) . message , 'ERR' , true ) ; } } private async buildCiTree ( buildsPattern , progress , checkCanceled ) { try { let projectKey = Configuration . getProjectKey ( ) ; const searchResult = await this . _treesManager . connectionManager . searchArtifactsByAql ( CiManager . createAqlForBuildArtifacts ( buildsPattern , projectKey ) ) ; if ( ! searchResult ) { throw new Error ( 'Failed searching for builds in Artifactory' ) ; } if ( ! searchResult . results . length ) { return ; } progress . report ( { message : `${ searchResult . results . length } builds` } ) ; const producerQueue = new PQueue ( { concurrency : 1 } ) ; const consumerQueue = new PQueue ( { concurrency : 1 } ) ; let xraySupported = await ConnectionUtils . testXrayVersionForCi ( this . _treesManager . connectionManager . createJfrogClient ( ) , this . _treesManager . logManager ) ; for ( const entry of searchResult . results ) { checkCanceled ( ) ; producerQueue . add ( ( ) => this . getBuildInfoAndProduce ( entry , consumerQueue , this . buildsCache , progress , checkCanceled , searchResult . results . length , xraySupported , projectKey ) ) ; } await producerQueue . onEmpty ( ) ; await consumerQueue . onEmpty ( ) ; } catch ( error ) { if ( error instanceof ScanCancellationError ) { throw error ; } vscode . window . showErrorMessage ( ( < any > error ) . message ) ; } } private async getBuildInfoAndProduce ( searchEntry , consumerQueue , buildsCache , progress , checkCanceled , buildsNum , xraySupported , projectKey ) { try { checkCanceled ( ) ; const buildName = decodeURIComponent ( searchEntry . path ) ; const split = searchEntry . name . split ( '-' ) ; if ( split . length !== 2 ) { throw new Error ( 'unexpected build artifact name' ) ; } const buildNumber = split [ 0 ] ; const timestamp = split [ 1 ] ; let build = buildsCache . loadBuildInfo ( timestamp , buildName , buildNumber , projectKey ) || ( await this . downloadBuildInfo ( searchEntry , buildsCache , projectKey ) ) ; let buildGeneralInfo = BuildsUtils . createBuildGeneralInfo ( build , this . _treesManager . logManager ) ; if ( ! build ) { progress . report ( { message : `${ buildsNum } builds` , increment : 100 / ( buildsNum * 2 ) } ) ; return ; } this . addResults ( buildGeneralInfo ) ; consumerQueue . add ( ( ) => this . handleXrayBuildDetails ( buildGeneralInfo , buildsCache , progress , checkCanceled , buildsNum , xraySupported , projectKey ) ) ; } catch ( error ) { if ( error instanceof ScanCancellationError ) { throw error ; } vscode . window . showErrorMessage ( 'Could not download build info.' , < vscode . MessageOptions > { modal : false } ) ; this . _treesManager . logManager . logMessage ( ( < any > error ) . message , 'ERR' , true ) ; } finally { progress . report ( { message : `${ buildsNum } builds` , increment : 100 / ( buildsNum * 2 ) } ) ; } } private async downloadBuildInfo ( searchEntry , buildsCache , projectKey ) { const artifactPath = '/' + CiManager . getBuildInfoRepo ( projectKey ) + '/' + searchEntry . path + '/' + searchEntry . name ; try { const build = await this . _treesManager . connectionManager . downloadArtifact ( artifactPath ) ; const timestamp = Date . parse ( build . started ) . toString ( ) ; buildsCache . save ( JSON . stringify ( build ) , timestamp , build . name , build . number , projectKey , Type . BUILD_INFO ) ; return build ; } catch ( error ) { this . _treesManager . logManager . logMessage ( `Could not get build details from Artifactory for build: '${ searchEntry . path }/${ searchEntry . name }': ${ error }` , 'ERR' , true ) ; } } private async downloadBuildDetails ( buildGeneralInfo , buildsCache , projectKey ) { let timestamp = buildGeneralInfo . startedTimestamp ; let buildName = buildGeneralInfo . artifactId ; let buildNumber = buildGeneralInfo . version ; const detailsResponse = await this . _treesManager . connectionManager . downloadBuildDetails ( buildName , buildNumber , projectKey ) ; if ( ! detailsResponse . is_scan_completed || ! ! detailsResponse . error_details || ! detailsResponse . components ) { if ( ! ! detailsResponse . error_details ) { this . _treesManager . logManager . logMessage ( `Could not get build details from Xray for build: '${ buildName }/${ buildNumber }': ${ detailsResponse . error_details . error_message }` , 'ERR' , true ) ; } return ; } buildsCache . save ( JSON . stringify ( detailsResponse ) , timestamp , buildName , buildNumber , projectKey , Type . BUILD_SCAN_RESULTS ) ; } private addResults ( bgi ) { const buildTree = new BuildsNode ( bgi ) ; this . root . addChild ( buildTree ) ; } private async handleXrayBuildDetails ( buildGeneralInfo , buildsCache , progress , checkCanceled , buildsNum , xraySupported , projectKey ) { try { if ( ! xraySupported ) { return ; } const buildName = buildGeneralInfo . artifactId ; const buildNumber = buildGeneralInfo . version ; const timestamp = buildGeneralInfo . startedTimestamp ; checkCanceled ( ) ; if ( ! buildsCache . loadScanResults ( timestamp , buildName , buildNumber , projectKey ) ) { await this . downloadBuildDetails ( buildGeneralInfo , buildsCache , projectKey ) ; } } catch ( error ) { if ( error instanceof ScanCancellationError ) { throw error ; } this . _treesManager . logManager . logMessage ( 'Could not get build details from xray: ' + ( < any > error ) . message , 'ERR' , true ) ; } finally { progress . report ( { message : `${ buildsNum } builds` , increment : 100 / ( buildsNum * 2 ) } ) ; } } private static createAqlForBuildArtifacts ( buildsPattern , projectKey ) { const spaceReplacementPlaceholder = 'JFROG_VSCODE_EXTENSION_SPACE_PLACEHOLDER' ; buildsPattern = buildsPattern . replace ( / /g , spaceReplacementPlaceholder ) ; buildsPattern = encodeURIComponent ( buildsPattern ) ; buildsPattern = buildsPattern . replace ( new RegExp ( spaceReplacementPlaceholder , 'g' ) , ' ' ) ; buildsPattern = buildsPattern . replace ( /%/g , '?' ) ; return ( `items.find({` + `"repo":"${ CiManager . getBuildInfoRepo ( projectKey ) }",` + `"path":{"$match":"${ buildsPattern }"}})` + `.include("name","repo","path","created")` + `.sort({"$desc":["created"]})` + `.limit(${ CiManager . DISPLAY_BUILDS_NUM })` ) ; } private static getBuildInfoRepo ( repoKey ) { return ( repoKey || 'artifactory' ) + '-build-info' ; } } class IssuesAndLicensesPair { public _issues : Set < IssueIdWithSeverity > = new Set < IssueIdWithSeverity > ( ) ; public _licenses : Set < string > = new Set < string > ( ) ; } class IssueIdWithSeverity { constructor ( public _issueId , public _severity ) { } }