import tl = require ( 'azure-pipelines-task-lib/task' ) ; import fs = require ( 'fs' ) ; import iconv = require ( 'iconv-lite' ) ; import jschardet = require ( 'jschardet' ) ; import path = require ( 'path' ) ; import os = require ( 'os' ) ; import crypto = require ( 'crypto' ) ; import trackEvent , { TelemetryEvent } from './telemetry' ; import yaml = require ( 'js-yaml' ) ; const ENCODING_AUTO = 'auto' ; const ENCODING_ASCII = 'ascii' ; const ENCODING_UTF_7 = 'utf-7' ; const ENCODING_UTF_8 = 'utf-8' ; const ENCODING_UTF_16LE = 'utf-16le' ; const ENCODING_UTF_16BE = 'utf-16be' ; const ENCODING_WIN1252 = 'windows1252' ; const ENCODING_ISO_8859_1 = 'iso88591' ; const ACTION_WARN = 'warn' ; const ACTION_FAIL = 'fail' ; const XML_ESCAPE = /[<>&'"]/g ; const JSON_ESCAPE = /["\\\b\f\n\r\t]/g ; const WIN32_DIRECTORY_SEPARATOR = /\\/g ; const POSIX_DIRECTORY_SEPARATOR = /\//g ; const OUTPUT_WILDCARD = /\*/g ; interface Options { readonly encoding : string ; readonly keepToken : boolean ; readonly actionOnMissing : string ; readonly writeBOM : boolean ; readonly emptyValue : string ; readonly escapeType : string ; readonly escapeChar : string ; readonly charsToEscape : string ; readonly verbosity : string ; readonly defaultValue : string ; readonly enableTransforms : boolean ; } interface Rule { isInputWildcard : boolean ; inputPatterns : string [ ] ; isOutputRelative : boolean ; outputPattern : string ; } interface ILogger { debug ( message ) : void ; info ( message ) : void ; warn ( message ) : void ; error ( message ) : void ; } class NullLogger implements ILogger { public debug ( message ) { } public info ( message ) { } public warn ( message ) { } public error ( message ) { } } enum LogLevel { Debug , Info , Warn , Error , Off = 255 } class Logger implements ILogger { private _level : LogLevel ; constructor ( level ) { this . _level = level ; } public debug ( message ) { this . log ( LogLevel . Debug , message ) ; } public info ( message ) { this . log ( LogLevel . Info , message ) ; } public warn ( message ) { this . log ( LogLevel . Warn , message ) ; } public error ( message ) { this . log ( LogLevel . Error , message ) ; } private log ( level , message ) { if ( level === LogLevel . Debug ) tl . debug ( message ) ; if ( level === LogLevel . Error ) tl . setResult ( tl . TaskResult . Failed , message ) ; if ( level < this . _level ) return ; switch ( level ) { case LogLevel . Debug : case LogLevel . Info : console . log ( message ) ; break ; case LogLevel . Warn : tl . warning ( message ) ; break ; } } } class Counter { public Tokens : number = 0 ; public Replaced : number = 0 ; public Files : number = 0 ; public DefaultValues : number = 0 ; public Transforms : number = 0 ; } var logger = new NullLogger ( ) ; var globalCounters = new Counter ( ) ; var fileVariables = { } ; var mapEncoding = function ( encoding ) : string { switch ( encoding ) { case 'auto' : return ENCODING_AUTO ; case 'Ascii' : case 'ascii' : return ENCODING_ASCII ; case 'UTF7' : case 'utf-7' : return ENCODING_UTF_7 ; case 'UTF8' : case 'utf-8' : return ENCODING_UTF_8 ; case 'Unicode' : case 'utf-16le' : return ENCODING_UTF_16LE ; case 'BigEndianUnicode' : case 'utf-16be' : return ENCODING_UTF_16BE ; case 'win1252' : return ENCODING_WIN1252 ; case 'iso88591' : return ENCODING_ISO_8859_1 ; case 'UTF32' : throw new Error ( 'utf-32 encoding is no more supported.' ) ; case 'BigEndianUTF32' : throw new Error ( 'utf-32be encoding is no more supported.' ) ; default : throw new Error ( 'invalid encoding: ' + encoding ) ; } } ; var getEncoding = function ( filePath ) : string { let buffer = fs . readFileSync ( filePath , { flag : 'r' } ) ; let charset = jschardet . detect ( buffer ) ; switch ( charset . encoding ) { case 'ascii' : return ENCODING_ASCII ; case 'UTF-8' : return ENCODING_UTF_8 ; case 'UTF-16LE' : return ENCODING_UTF_16LE ; case 'UTF-16BE' : return ENCODING_UTF_16BE ; case 'windows-1252' : return ENCODING_WIN1252 ; default : return ENCODING_ASCII ; } } ; var loadVariablesFromJson = function ( value , name , separator , variables ) : number { let count = 0 ; let type = typeof ( value ) ; let prefix = name ; if ( name . length != 0 ) prefix += separator ; if ( value === null || type == "boolean" || type == "number" || type == "string" ) { variables [ name ] = ( value === null ? "" : value ) + "" ; ++ count ; logger . debug ( '  loaded variable: ' + name ) ; } else if ( Array . isArray ( value ) ) { value . forEach ( ( v , i ) => { count += loadVariablesFromJson ( v , prefix + i , separator , variables ) ; } ) ; } else if ( type == "object" ) { Object . keys ( value ) . forEach ( key => { count += loadVariablesFromJson ( value [ key ] , prefix + key , separator , variables ) ; } ) ; } return count ; } ; var replaceTokensInFile = function ( filePath , outputPath , regex , transformRegex , options ) : void { logger . info ( '##[group]replacing tokens in: ' + filePath ) ; if ( filePath !== outputPath ) logger . info ( '  output in: ' + outputPath ) ; let encoding = options . encoding ; if ( options . encoding === ENCODING_AUTO ) encoding = getEncoding ( filePath ) ; logger . debug ( '  using encoding: ' + encoding ) ; let localCounter = new Counter ( ) ; let content = iconv . decode ( fs . readFileSync ( filePath ) , encoding ) ; content = content . replace ( regex , ( match , name ) => { ++ localCounter . Tokens ; let transformName = null ; if ( options . enableTransforms ) { let m = name . match ( transformRegex ) ; if ( m ) { ++ localCounter . Transforms ; transformName = m [ 1 ] ; name = m [ 2 ] ; } } let value = tl . getVariable ( name ) ; if ( name in fileVariables ) value = fileVariables [ name ] ; let usedDefaultValue = false ; if ( ! value && options . defaultValue ) { ++ localCounter . DefaultValues ; value = options . defaultValue ; usedDefaultValue = true ; } if ( ! value ) { if ( options . keepToken ) value = match ; else value = '' ; let message = '  variable not found: ' + name ; switch ( options . actionOnMissing ) { case ACTION_WARN : logger . warn ( message ) ; break ; case ACTION_FAIL : logger . error ( message ) ; break ; default : logger . debug ( message ) ; } } else { ++ localCounter . Replaced ; if ( options . emptyValue && value === options . emptyValue ) value = '' ; if ( transformName ) { switch ( transformName ) { case 'lower' : value = value . toLowerCase ( ) ; break ; case 'upper' : value = value . toUpperCase ( ) ; break ; case 'noescape' : break ; case 'base64' : value = Buffer . from ( value ) . toString ( 'base64' ) ; break ; default : -- localCounter . Transforms ; logger . warn ( '  unknown transform: ' + transformName ) ; break ; } } } let escapeType = options . escapeType ; if ( escapeType === 'auto' ) { switch ( path . extname ( filePath ) ) { case '.json' : escapeType = 'json' ; break ; case '.xml' : escapeType = 'xml' ; break ; default : escapeType = 'none' ; break ; } } if ( transformName === 'noescape' ) { escapeType = 'none' ; } logger . debug ( '  ' + name + ': ' + value + ( usedDefaultValue ? ' (default value)' : '' ) ) ; switch ( escapeType ) { case 'json' : value = value . replace ( JSON_ESCAPE , match => { switch ( match ) { case '"' : case '\\' : return '\\' + match ; case '\b' : return "\\b" ; case '\f' : return "\\f" ; case '\n' : return "\\n" ; case '\r' : return "\\r" ; case '\t' : return "\\t" ; } } ) ; break ; case 'xml' : value = value . replace ( XML_ESCAPE , match => { switch ( match ) { case '<' : return '&lt;' ; case '>' : return '&gt;' ; case '&' : return '&amp;' ; case '\'' : return '&apos;' ; case '"' : return '&quot;' ; } } ) ; break ; case 'custom' : if ( options . escapeChar && options . charsToEscape ) for ( var c of options . charsToEscape ) value = value . split ( c ) . join ( options . escapeChar + c ) ; break ; } return value ; } ) ; let mkdirSyncRecursive = function ( p ) { if ( fs . existsSync ( p ) ) return ; mkdirSyncRecursive ( path . dirname ( p ) ) ; fs . mkdirSync ( p ) ; logger . debug ( '  created folder: ' + p ) ; } ; mkdirSyncRecursive ( path . dirname ( path . resolve ( outputPath ) ) ) ; if ( localCounter . Tokens ) { fs . writeFileSync ( outputPath , iconv . encode ( content , encoding , { addBOM : options . writeBOM , stripBOM : null , defaultEncoding : null } ) ) ; } else if ( filePath !== outputPath ) { fs . writeFileSync ( outputPath , fs . readFileSync ( filePath ) ) ; } logger . info ( '  ' + localCounter . Replaced + ' token(s) replaced out of ' + localCounter . Tokens + ( localCounter . DefaultValues ? ' (using ' + localCounter . DefaultValues + ' default value(s))' : '' ) + ( options . enableTransforms ? ' and running ' + localCounter . Transforms + ' transformation(s)' : '' ) ) ; logger . info ( '##[endgroup]' ) ; globalCounters . Tokens += localCounter . Tokens ; globalCounters . Replaced += localCounter . Replaced ; globalCounters . DefaultValues += localCounter . DefaultValues ; globalCounters . Transforms += localCounter . Transforms ; } ; var mapLogLevel = function ( level ) : LogLevel { switch ( level ) { case "normal" : return LogLevel . Info ; case "detailed" : return LogLevel . Debug ; case "off" : return LogLevel . Off ; } return LogLevel . Info ; } ; var normalize = function ( p ) : string { return os . platform ( ) === 'win32' ? p . replace ( POSIX_DIRECTORY_SEPARATOR , '\\' ) : p . replace ( WIN32_DIRECTORY_SEPARATOR , '/' ) ; } ; async function run ( ) { let startTime = new Date ( ) ; let serverType = tl . getVariable ( 'System.ServerType' ) ; let telemetryEnabled = tl . getBoolInput ( 'enableTelemetry' , true ) && tl . getVariable ( 'REPLACETOKENS_DISABLE_TELEMETRY' ) !== 'true' ; let proxyUrl = undefined ; let config = tl . getHttpProxyConfiguration ( ) ; if ( config ) proxyUrl = config . proxyUrl ; let telemetryEvent = { account : crypto . createHash ( 'sha256' ) . update ( tl . getVariable ( 'system.collectionid' ) ) . digest ( 'hex' ) , pipeline : crypto . createHash ( 'sha256' ) . update ( tl . getVariable ( 'system.teamprojectid' ) + tl . getVariable ( 'system.definitionid' ) ) . digest ( 'hex' ) , pipelineType : tl . getVariable ( 'release.releaseid' ) ? 'release' : 'build' , serverType : ! serverType || serverType . toLowerCase ( ) !== 'hosted' ? 'server' : 'services' , } as TelemetryEvent ; try { let root = tl . getPathInput ( 'rootDirectory' , false , true ) ; let tokenPattern = tl . getInput ( 'tokenPattern' , true ) ; let tokenPrefix = tl . getInput ( 'tokenPrefix' , true ) ; let tokenSuffix = tl . getInput ( 'tokenSuffix' , true ) ; let useLegacyPattern = tl . getBoolInput ( 'useLegacyPattern' , true ) ; let options = { encoding : mapEncoding ( tl . getInput ( 'encoding' , true ) ) , keepToken : tl . getBoolInput ( 'keepToken' , true ) , actionOnMissing : tl . getInput ( 'actionOnMissing' , true ) , writeBOM : tl . getBoolInput ( 'writeBOM' , true ) , emptyValue : tl . getInput ( 'emptyValue' , false ) , escapeType : tl . getInput ( 'escapeType' , false ) , escapeChar : tl . getInput ( 'escapeChar' , false ) , defaultValue : tl . getInput ( 'defaultValue' , false ) , charsToEscape : tl . getInput ( 'charsToEscape' , false ) , verbosity : tl . getInput ( 'verbosity' , true ) , enableTransforms : tl . getBoolInput ( 'enableTransforms' , false ) , } ; let transformPrefix = tl . getInput ( 'transformPrefix' , true ) ; let transformSuffix = tl . getInput ( 'transformSuffix' , true ) ; let actionOnNoFiles = tl . getInput ( 'actionOnNoFiles' , true ) ; logger = new Logger ( mapLogLevel ( options . verbosity ) ) ; let rules = [ ] ; let ruleUsingInputWildcardCount = 0 ; let ruleUsingNegativeInputPattern = 0 ; let ruleUsingOutputPatternCount = 0 ; tl . getDelimitedInput ( 'targetFiles' , '\n' , true ) . forEach ( ( l ) => { if ( l ) l . split ( ',' ) . forEach ( ( line ) => { if ( line ) { let ruleParts = line . split ( '=>' ) ; let rule = { isInputWildcard : false , inputPatterns : normalize ( ruleParts [ 0 ] . trim ( ) ) . split ( ';' ) , isOutputRelative : false , outputPattern : null } ; rule . isInputWildcard = path . basename ( rule . inputPatterns [ 0 ] ) . indexOf ( '*' ) != - 1 ; if ( ruleParts . length > 1 ) { rule . outputPattern = normalize ( ruleParts [ 1 ] . trim ( ) ) ; rule . isOutputRelative = ! path . isAbsolute ( rule . outputPattern ) ; } rules . push ( rule ) ; if ( ruleParts [ 0 ] . indexOf ( '!' ) != - 1 ) ++ ruleUsingNegativeInputPattern ; if ( rule . isInputWildcard ) ++ ruleUsingInputWildcardCount ; if ( rule . outputPattern ) ++ ruleUsingOutputPatternCount ; } } ) ; } ) ; let variableSeparator = tl . getInput ( 'variableSeparator' , false ) ; let variableFilesCount = 0 ; tl . getDelimitedInput ( 'variableFiles' , '\n' , false ) . forEach ( ( l ) => { if ( l ) l . split ( ',' ) . forEach ( ( p ) => { if ( p ) { tl . findMatch ( root , normalize ( p ) ) . forEach ( filePath => { if ( tl . stats ( filePath ) . isDirectory ( ) ) return ; if ( ! tl . exist ( filePath ) ) { logger . error ( 'file not found: ' + filePath ) ; return ; } logger . info ( '##[group]loading variables from: ' + filePath ) ; let encoding = getEncoding ( filePath ) ; let extension = path . extname ( filePath ) . toLowerCase ( ) ; let content = iconv . decode ( fs . readFileSync ( filePath ) , encoding ) ; let variables = extension === '.yaml' || extension === '.yml' ? yaml . load ( content ) : JSON . parse ( content ) ; let count = loadVariablesFromJson ( variables , '' , variableSeparator , fileVariables ) ; logger . info ( '  ' + count + ' variable(s) loaded.' ) ; logger . info ( '##[endgroup]' ) ; ++ variableFilesCount ; } ) ; } } ) ; } ) ; switch ( tokenPattern ) { case 'default' : tokenPrefix = '#{' ; tokenSuffix = '}#' ; break ; case 'rm' : tokenPrefix = '__' ; tokenSuffix = '__' ; break ; case 'octopus' : tokenPrefix = '#{' ; tokenSuffix = '}' ; break ; case 'azpipelines' : tokenPrefix = '$(' ; tokenSuffix = ')' ; break ; case 'doublebraces' : tokenPrefix = '{{' ; tokenSuffix = '}}' ; break ; } let escapedTokenPrefix = tokenPrefix . replace ( /[-\/\\^$*+?.()|[\]{}]/g , '\\$&' ) ; let escapedTokenSuffix = tokenSuffix . replace ( /[-\/\\^$*+?.()|[\]{}]/g , '\\$&' ) ; let pattern = useLegacyPattern ? escapedTokenPrefix + '((?:(?!' + escapedTokenSuffix + ').)*)' + escapedTokenSuffix : escapedTokenPrefix + '\\s*((?:(?!' + escapedTokenPrefix + ')(?!\\s*' + escapedTokenSuffix + ').)*)\\s*' + escapedTokenSuffix ; let regex = new RegExp ( pattern , 'gm' ) ; logger . debug ( 'pattern: ' + regex . source ) ; let escapedTransformPrefix = transformPrefix . replace ( /[-\/\\^$*+?.()|[\]{}]/g , '\\$&' ) ; let escapedTransformSuffix = transformSuffix . replace ( /[-\/\\^$*+?.()|[\]{}]/g , '\\$&' ) ; let transformPattern = '\\s*(.*)' + escapedTransformPrefix + '\\s*((?:(?!' + escapedTransformPrefix + ')(?!\\s*' + escapedTransformSuffix + ').)*)\\s*' + escapedTransformSuffix + '\\s*' ; let transformRegex = new RegExp ( transformPattern ) ; logger . debug ( 'transform pattern: ' + transformRegex . source ) ; telemetryEvent . actionOnMissing = options . actionOnMissing ; telemetryEvent . charsToEscape = options . charsToEscape ; telemetryEvent . emptyValue = options . emptyValue ; telemetryEvent . encoding = options . encoding ; telemetryEvent . escapeChar = options . escapeChar ; telemetryEvent . escapeType = options . escapeType ; telemetryEvent . keepToken = options . keepToken ; telemetryEvent . pattern = regex . source ; telemetryEvent . result = 'succeeded' ; telemetryEvent . rules = rules . length ; telemetryEvent . rulesWithInputWildcard = ruleUsingInputWildcardCount ; telemetryEvent . rulesWithNegativePattern = ruleUsingNegativeInputPattern ; telemetryEvent . rulesWithOutputPattern = ruleUsingOutputPatternCount ; telemetryEvent . tokenPrefix = tokenPrefix ; telemetryEvent . tokenSuffix = tokenSuffix ; telemetryEvent . variableFiles = variableFilesCount ; telemetryEvent . variableSeparator = variableSeparator ; telemetryEvent . verbosity = options . verbosity ; telemetryEvent . writeBOM = options . writeBOM ; telemetryEvent . useLegacyPattern = useLegacyPattern ; telemetryEvent . enableTransforms = options . enableTransforms ; telemetryEvent . transformPrefix = transformPrefix ; telemetryEvent . transformSuffix = transformSuffix ; telemetryEvent . transformPattern = transformPattern ; telemetryEvent . defaultValue = options . defaultValue ; telemetryEvent . tokenPattern = tokenPattern ; telemetryEvent . actionOnNoFiles = actionOnNoFiles ; rules . forEach ( rule => { tl . findMatch ( root , rule . inputPatterns ) . forEach ( filePath => { if ( tl . stats ( filePath ) . isDirectory ( ) ) return ; if ( ! tl . exist ( filePath ) ) { logger . error ( 'file not found: ' + filePath ) ; return ; } let outputPath = filePath ; if ( rule . outputPattern ) { outputPath = rule . outputPattern ; if ( rule . isInputWildcard ) { let inputBasename = path . basename ( rule . inputPatterns [ 0 ] ) ; let inputWildcardIndex = inputBasename . indexOf ( '*' ) ; let fileBasename = path . basename ( filePath ) ; let token = fileBasename . substring ( inputWildcardIndex , fileBasename . length - ( inputBasename . length - inputWildcardIndex - 1 ) ) ; outputPath = outputPath . replace ( OUTPUT_WILDCARD , token ) ; } if ( rule . isOutputRelative ) outputPath = path . join ( path . dirname ( filePath ) , outputPath ) ; } replaceTokensInFile ( filePath , outputPath , regex , transformRegex , options ) ; ++ globalCounters . Files ; } ) ; } ) ; tl . setVariable ( 'tokenReplacedCount' , globalCounters . Replaced . toString ( ) ) ; tl . setVariable ( 'tokenFoundCount' , globalCounters . Tokens . toString ( ) ) ; tl . setVariable ( 'fileProcessedCount' , globalCounters . Files . toString ( ) ) ; tl . setVariable ( 'transformExecutedCount' , globalCounters . Transforms . toString ( ) ) ; tl . setVariable ( 'defaultValueCount' , globalCounters . DefaultValues . toString ( ) ) ; let duration = ( + new Date ( ) - ( + startTime ) ) / 1000 ; if ( globalCounters . Files === 0 && actionOnNoFiles === ACTION_WARN ) { logger . warn ( 'found no files to process in ' + duration + ' seconds.' ) ; } else if ( globalCounters . Files === 0 && actionOnNoFiles === ACTION_FAIL ) { logger . error ( 'found no files to process in ' + duration + ' seconds.' ) ; } else { logger . info ( 'replaced ' + globalCounters . Replaced + ' tokens out of ' + globalCounters . Tokens + ( globalCounters . DefaultValues ? ' (using ' + globalCounters . DefaultValues + ' default value(s))' : '' ) + ( options . enableTransforms ? ' and running ' + globalCounters . Transforms + ' functions' : '' ) + ' in ' + globalCounters . Files + ' file(s) in ' + duration + ' seconds.' ) ; } telemetryEvent . duration = duration ; telemetryEvent . tokenReplaced = globalCounters . Replaced ; telemetryEvent . tokenFound = globalCounters . Tokens ; telemetryEvent . defaultValueReplaced = globalCounters . DefaultValues ; telemetryEvent . fileProcessed = globalCounters . Files ; telemetryEvent . transformExecuted = globalCounters . Transforms ; } catch ( err ) { telemetryEvent . result = 'failed' ; logger . error ( err . message ) ; } finally { if ( telemetryEnabled ) { var trackedData = trackEvent ( telemetryEvent , proxyUrl ) ; tl . debug ( 'sent usage telemetry: ' + JSON . stringify ( trackedData ) ) ; } } } run ( ) ;