import { blake2AsHex } from "@polkadot/util-crypto" ; import ApiHandler from "./ApiHandler" ; import ChainData from "./chaindata" ; import { WEEK , KOTVBackendEndpoint , SIXTEEN_HOURS } from "./constants" ; import logger from "./logger" ; import { CandidateData } from "./types" ; import axios from "axios" ; import { formatAddress } from "./util" ; import { Config } from "./config" ; import Db from "./db" ; import Database from "./db" ; import semver from "semver" ; import { absMax , absMin , asc , getStats , mean , median , q10 , q25 , q75 , q90 , scaled , scaledDefined , std , } from "./score" ; export interface Constraints { processCandidates ( candidates ) : Promise < [ Set < any > , Set < any > ] > ; } export class OTV implements Constraints { private chaindata : ChainData ; private skipConnectionTime : boolean ; private skipIdentity : boolean ; private skipStakedDesitnation : boolean ; private skipClientUpgrade : boolean ; private skipUnclaimed : boolean ; private minSelfStake : number ; private commission : number ; private unclaimedEraThreshold : number ; private config : Config ; private db : Database ; private validCache : CandidateData [ ] = [ ] ; private invalidCache : CandidateData [ ] = [ ] ; private validMapCache : Map < string , CandidateData > = new Map ( ) ; private invalidMapCache : Map < string , CandidateData > = new Map ( ) ; constructor ( handler , skipConnectionTime = false , skipIdentity = false , skipStakedDestination = false , skipClientUpgrade = false , skipUnclaimed = false , minSelfStake = 0 , commission = 0 , unclaimedEraThreshold = 0 , config , db ) { this . chaindata = new ChainData ( handler ) ; this . skipConnectionTime = skipConnectionTime ; this . skipIdentity = skipIdentity ; this . skipStakedDesitnation = skipStakedDestination ; this . skipClientUpgrade = skipClientUpgrade ; this . skipUnclaimed = skipUnclaimed ; this . minSelfStake = minSelfStake ; this . commission = commission ; this . unclaimedEraThreshold = unclaimedEraThreshold ; this . config = config ; this . db = db ; } get validCandidateCache ( ) : CandidateData [ ] { return this . validCache ; } get invalidCandidateCache ( ) : CandidateData [ ] { return this . invalidCache ; } addToValidCache ( address , candidate ) { if ( this . invalidMapCache . has ( address ) ) { this . invalidMapCache . delete ( address ) ; } if ( this . validMapCache . has ( address ) ) { return ; } else { this . validMapCache . set ( address , candidate ) ; } } addToInvalidCache ( address , candidate ) { if ( this . validMapCache . has ( address ) ) { this . validMapCache . delete ( address ) ; } if ( this . invalidMapCache . has ( address ) ) { return ; } else { this . invalidMapCache . set ( address , candidate ) ; } } async checkCandidateStash ( address ) { const candidate = await this . db . getCandidate ( address ) ; return await this . checkCandidate ( candidate ) ; } async checkCandidate ( candidate ) { let valid = false ; const onlineValid = await checkOnline ( this . db , candidate ) ; const validateValid = await checkValidateIntention ( this . config , this . chaindata , this . db , candidate ) ; const versionValid = await checkLatestClientVersion ( this . config , this . db , candidate ) ; const monitoringWeekValid = await checkConnectionTime ( this . config , this . db , candidate ) ; const identityValid = await checkIdentity ( this . chaindata , this . db , candidate ) ; const offlineValid = await checkOffline ( this . db , candidate ) ; let rewardDestinationValid = true ; if ( ! this . skipStakedDesitnation ) { rewardDestinationValid = ( await checkRewardDestination ( this . db , this . chaindata , candidate ) ) || false ; } const commissionValid = ( await checkCommission ( this . db , this . chaindata , this . commission , candidate ) ) || false ; const selfStakeValid = ( await checkSelfStake ( this . db , this . chaindata , this . minSelfStake , candidate ) ) || false ; const unclaimedValid = ( await checkUnclaimed ( this . db , this . chaindata , this . unclaimedEraThreshold , candidate ) ) || false ; const blockedValid = ( await checkBlocked ( this . db , this . chaindata , candidate ) ) || false ; let kusamaValid = true ; try { if ( ! ! candidate . kusamaStash ) { kusamaValid = ( await checkKusamaRank ( this . db , candidate ) ) || false ; } } catch ( e ) { logger . info ( `Error trying to get kusama data...` ) ; } valid = onlineValid && validateValid && versionValid && monitoringWeekValid && identityValid && offlineValid && rewardDestinationValid && commissionValid && selfStakeValid && unclaimedValid && blockedValid && kusamaValid ; await this . db . setValid ( candidate . stash , valid ) ; if ( valid ) { this . addToValidCache ( candidate . stash , candidate ) ; this . db . setLastValid ( candidate . stash ) ; } else { this . addToInvalidCache ( candidate . stash , candidate ) ; } return valid ; } async scoreAllCandidates ( ) { const candidates = await this . db . allCandidates ( ) ; await this . scoreCandidates ( candidates , this . db ) ; } async scoreCandidates ( candidates , db ) { let rankedCandidates = [ ] ; const validCandidates = candidates . filter ( ( candidate ) => candidate . valid ) ; if ( validCandidates . length < 2 ) return ; const bondedValues = validCandidates . map ( ( candidate ) => { return candidate . bonded ? candidate . bonded : 0 ; } ) ; const bondedStats = bondedValues . length > 0 ? getStats ( bondedValues ) : [ ] ; const faultsValues = validCandidates . map ( ( candidate ) => { return candidate . faults ? candidate . faults : 0 ; } ) ; const faultsStats = getStats ( faultsValues ) ; const inclusionValues = validCandidates . map ( ( candidate ) => { return candidate . inclusion ? candidate . inclusion : 0 ; } ) ; const inclusionStats = getStats ( inclusionValues ) ; const spanInclusionValues = validCandidates . map ( ( candidate ) => { return candidate . spanInclusion ? candidate . spanInclusion : 0 ; } ) ; const spanInclusionStats = getStats ( spanInclusionValues ) ; const discoveredAtValues = validCandidates . map ( ( candidate ) => { return candidate . discoveredAt ? candidate . discoveredAt : 0 ; } ) ; const discoveredAtStats = getStats ( discoveredAtValues ) ; const nominatedAtValues = validCandidates . map ( ( candidate ) => { return candidate . nominatedAt ? candidate . nominatedAt : 0 ; } ) ; const nominatedAtStats = getStats ( nominatedAtValues ) ; const offlineValues = validCandidates . map ( ( candidate ) => { return candidate . offlineAccumulated ? candidate . offlineAccumulated : 0 ; } ) ; const offlineStats = getStats ( offlineValues ) ; const rankValues = validCandidates . map ( ( candidate ) => { return candidate . rank ? candidate . rank : 0 ; } ) ; const rankStats = getStats ( rankValues ) ; const unclaimedValues = validCandidates . map ( ( candidate ) => { return candidate . unclaimedEras && candidate . unclaimedEras . length ? candidate . unclaimedEras . length : 0 ; } ) ; const unclaimedStats = getStats ( unclaimedValues ) ; const locationMap = new Map ( ) ; const locationArr = [ ] ; for ( const candidate of validCandidates ) { const location = candidate . location || "No Location" ; const locationCount = locationMap . get ( location ) ; if ( ! locationCount ) { locationMap . set ( location , 1 ) ; } else { locationMap . set ( location , locationCount + 1 ) ; } } for ( const location of locationMap . entries ( ) ) { const [ name , numberOfNodes ] = location ; locationArr . push ( { name , numberOfNodes } ) ; } const locationValues = locationArr . map ( ( location ) => { return location . numberOfNodes ; } ) ; const locationStats = getStats ( locationValues ) ; const councilStakeValues = validCandidates . map ( ( candidate ) => { return candidate . councilStake ? candidate . councilStake : 0 ; } ) ; const councilStakeStats = getStats ( councilStakeValues ) ; const democracyValues = validCandidates . map ( ( candidate ) => { return candidate . democracyVoteCount ; } ) ; const democracyStats = getStats ( democracyValues ) ; await db . setValidatorScoreMetadata ( bondedStats , this . BONDED_WEIGHT , faultsStats , this . FAULTS_WEIGHT , inclusionStats , this . INCLUSION_WEIGHT , spanInclusionStats , this . SPAN_INCLUSION_WEIGHT , discoveredAtStats , this . DISCOVERED_WEIGHT , nominatedAtStats , this . NOMINATED_WEIGHT , offlineStats , this . OFFLINE_WEIGHT , rankStats , this . RANK_WEIGHT , unclaimedStats , this . UNCLAIMED_WEIGHT , locationStats , this . LOCATION_WEIGHT , councilStakeStats , this . COUNCIL_WEIGHT , democracyStats , this . DEMOCRACY_WEIGHT , Date . now ( ) ) ; for ( const candidate of validCandidates ) { const scaledInclusion = scaledDefined ( candidate . inclusion , inclusionValues , 0.2 , 0.75 ) ; const inclusionScore = ( 1 - scaledInclusion ) * this . INCLUSION_WEIGHT ; const scaledSpanInclusion = scaledDefined ( candidate . spanInclusion , spanInclusionValues , 0.2 , 0.75 ) ; const spanInclusionScore = ( 1 - scaledSpanInclusion ) * this . SPAN_INCLUSION_WEIGHT ; const scaledDiscovered = scaled ( candidate . discoveredAt , discoveredAtValues ) ; const discoveredScore = ( 1 - scaledDiscovered ) * this . DISCOVERED_WEIGHT ; const scaledNominated = scaled ( candidate . nominatedAt , nominatedAtValues ) ; const nominatedScore = ( 1 - scaledNominated ) * this . NOMINATED_WEIGHT ; const scaledRank = scaled ( candidate . rank , rankValues ) ; const rankScore = scaledRank * this . RANK_WEIGHT ; const unclaimedScore = candidate . unclaimedEras ? - 1 * candidate . unclaimedEras . length * this . UNCLAIMED_WEIGHT : 0 ; const scaledBonded = scaledDefined ( candidate . bonded ? candidate . bonded : 0 , bondedValues , 0.05 , 0.85 ) ; const bondedScore = scaledBonded * this . BONDED_WEIGHT ; const scaledOffline = scaled ( candidate . offlineAccumulated , offlineValues ) ; const offlineScore = ( 1 - scaledOffline ) * this . OFFLINE_WEIGHT ; const scaledFaults = scaled ( candidate . faults , faultsValues ) ; const faultsScore = ( 1 - scaledFaults ) * this . FAULTS_WEIGHT ; const filteredLocation = locationArr . filter ( ( location ) => { if ( candidate . location == location . name ) return location . numberOfNodes ; } ) ; const candidateLocation = filteredLocation [ 0 ] ?. numberOfNodes ; const scaledLocation = scaledDefined ( candidateLocation , locationValues , 0.1 , 0.95 ) ; const locationScore = ( 1 - scaledLocation ) * this . LOCATION_WEIGHT || 0 ; const denom = await this . chaindata . getDenom ( ) ; const formattedBonded = candidate . bonded / denom ; const councilStakeScore = candidate . councilStake == 0 ? 0 : candidate . councilStake >= 0.75 * formattedBonded ? this . COUNCIL_WEIGHT : candidate . councilStake >= 0.5 * formattedBonded ? 0.75 * this . COUNCIL_WEIGHT : candidate . councilStake >= 0.25 * formattedBonded ? 0.5 * this . COUNCIL_WEIGHT : candidate . councilStake < 0.25 * formattedBonded ? 0.25 * this . COUNCIL_WEIGHT : 0 ; const democracyScore = candidate . democracyVoteCount * this . DEMOCRACY_WEIGHT ; const aggregate = inclusionScore + spanInclusionScore + faultsScore + discoveredScore + nominatedScore + rankScore + unclaimedScore + bondedScore + locationScore + councilStakeScore + democracyScore + offlineScore ; const randomness = 1 + Math . random ( ) * 0.15 ; const score = { total : aggregate * randomness , aggregate : aggregate , inclusion : inclusionScore , spanInclusion : spanInclusionScore , discovered : discoveredScore , nominated : nominatedScore , rank : rankScore , unclaimed : unclaimedScore , bonded : bondedScore , faults : faultsScore , offline : offlineScore , location : locationScore , councilStake : councilStakeScore , democracy : democracyScore , randomness : randomness , updated : Date . now ( ) , } ; await db . setValidatorScore ( candidate . stash , score . updated , score . total , score . aggregate ? score . aggregate : 0 , score . inclusion , score . spanInclusion , score . discovered , score . nominated , score . rank , score . unclaimed , score . bonded , score . faults , score . offline , score . location , score . councilStake , score . democracy , score . randomness ) ; const rankedCandidate = { aggregate : score , discoveredAt : candidate . discoveredAt , rank : candidate . rank , unclaimedEras : candidate . unclaimedEras , inclusion : candidate . inclusion , spanInclusion : candidate . spanInclusion , name : candidate . name , stash : candidate . stash , identity : candidate . identity , nominatedAt : candidate . nominatedAt , bonded : candidate . bonded , location : { location : candidate . location , otherNodes : candidateLocation , } , } ; rankedCandidates . push ( rankedCandidate ) ; } rankedCandidates = rankedCandidates . sort ( ( a , b ) => { return b . aggregate . total - a . aggregate . total ; } ) ; this . validCache = rankedCandidates ; return rankedCandidates ; } INCLUSION_WEIGHT = 40 ; SPAN_INCLUSION_WEIGHT = 60 ; DISCOVERED_WEIGHT = 5 ; NOMINATED_WEIGHT = 10 ; RANK_WEIGHT = 5 ; UNCLAIMED_WEIGHT = 10 ; BONDED_WEIGHT = 50 ; FAULTS_WEIGHT = 5 ; OFFLINE_WEIGHT = 2 ; LOCATION_WEIGHT = 40 ; COUNCIL_WEIGHT = 50 ; DEMOCRACY_WEIGHT = 10 ; async processCandidates ( candidates ) { logger . info ( `(OTV::processCandidates) Processing candidates` ) ; const [ activeEraIndex , eraErr ] = await this . chaindata . getActiveEraIndex ( ) ; if ( eraErr ) { throw eraErr ; } const good = new Set ( ) ; const bad = new Set ( ) ; for ( const candidate of candidates ) { if ( ! candidate ) { logger . info ( `{Constraints::processCandidates} candidate is null. Skipping..` ) ; continue ; } const { name , offlineSince , stash , skipSelfStake , offlineAccumulated , unclaimedEras , } = candidate ; const [ commission , err ] = await this . chaindata . getCommission ( stash ) ; if ( err ) { const reason = `${ name } ${ err }` ; logger . info ( reason ) ; bad . add ( { candidate , reason } ) ; continue ; } if ( commission > this . commission ) { const reason = `${ name } found commission higher than ten percent: ${ commission }` ; logger . info ( reason ) ; bad . add ( { candidate , reason } ) ; continue ; } if ( ! skipSelfStake ) { const [ bondedAmt , err2 ] = await this . chaindata . getBondedAmount ( stash ) ; if ( err2 ) { const reason = `${ name } ${ err2 }` ; logger . info ( reason ) ; bad . add ( { candidate , reason } ) ; continue ; } if ( bondedAmt < this . minSelfStake ) { const reason = `${ name } has less than the minimum required amount bonded: ${ bondedAmt }` ; logger . info ( reason ) ; bad . add ( { candidate , reason } ) ; continue ; } } if ( ! this . skipStakedDesitnation ) { const isStaked = await this . chaindata . destinationIsStaked ( stash ) ; if ( ! isStaked ) { const reason = `${ name } does not have reward destination set to Staked` ; bad . add ( { candidate , reason } ) ; continue ; } } const totalOffline = offlineAccumulated / WEEK ; if ( totalOffline > 0.02 ) { const reason = `${ name } has been offline ${ offlineAccumulated / 1000 / 60 } minutes this week.` ; logger . info ( reason ) ; bad . add ( { candidate , reason } ) ; continue ; } good . add ( candidate ) ; } return [ good , bad ] ; } } export const checkOnline = async ( db , candidate ) => { if ( ( candidate && Number ( candidate . onlineSince ) === 0 ) || Number ( candidate . offlineSince ) !== 0 ) { await db . setOnlineValidity ( candidate . stash , false ) ; return false ; } else { await db . setOnlineValidity ( candidate . stash , true ) ; return true ; } } ; export const checkValidateIntention = async ( config , chaindata , db , candidate ) => { const validators = await chaindata . getValidators ( ) ; if ( ! validators . includes ( formatAddress ( candidate ?. stash , config ) ) ) { db . setValidateIntentionValidity ( candidate . stash , false ) ; return false ; } else { db . setValidateIntentionValidity ( candidate . stash , true ) ; return true ; } } ; export const checkAllValidateIntentions = async ( config , chaindata , db , candidates ) => { const validators = await chaindata . getValidators ( ) ; for ( const candidate of candidates ) { if ( ! validators . includes ( formatAddress ( candidate . stash , config ) ) ) { db . setValidateIntentionValidity ( candidate . stash , false ) ; } else { db . setValidateIntentionValidity ( candidate . stash , true ) ; } } } ; export const checkLatestClientVersion = async ( config , db , candidate ) => { if ( ! config . constraints . skipClientUpgrade ) { const forceLatestRelease = config . constraints . forceClientVersion ; const latestRelease = await db . getLatestRelease ( ) ; if ( candidate . version && latestRelease && Date . now ( ) > latestRelease . publishedAt + SIXTEEN_HOURS ) { const nodeVersion = semver . coerce ( candidate . version ) ; const latestVersion = forceLatestRelease ? semver . clean ( forceLatestRelease ) : semver . clean ( latestRelease . name ) ; const isUpgraded = semver . gte ( nodeVersion , latestVersion ) ; if ( ! isUpgraded ) { db . setLatestClientReleaseValidity ( candidate . stash , false ) ; return false ; } else { db . setLatestClientReleaseValidity ( candidate . stash , true ) ; return true ; } } else { logger . warn ( `{latestRelease} Could not set release validity for ${ candidate . name } - version: ${ candidate . version } Latest release: ${ latestRelease . name } now: ${ Date . now ( ) }` ) ; return true ; } } else { db . setLatestClientReleaseValidity ( candidate . stash , true ) ; return true ; } } ; export const checkConnectionTime = async ( config , db , candidate ) => { if ( ! config . constraints . skipConnectionTime ) { const now = new Date ( ) . getTime ( ) ; if ( now - candidate . discoveredAt < WEEK ) { db . setConnectionTimeInvalidity ( candidate . stash , false ) ; return false ; } else { db . setConnectionTimeInvalidity ( candidate . stash , true ) ; return true ; } } else { db . setConnectionTimeInvalidity ( candidate . stash , true ) ; return true ; } } ; export const checkIdentity = async ( chaindata , db , candidate ) => { const [ hasIdentity , verified ] = await chaindata . hasIdentity ( candidate . stash ) ; if ( ! hasIdentity ) { const invalidityString = `${ candidate . name } does not have an identity set.` ; db . setIdentityInvalidity ( candidate . stash , false , invalidityString ) ; return false ; } if ( ! verified ) { const invalidityString = `${ candidate . name } has an identity but is not verified by the registrar.` ; db . setIdentityInvalidity ( candidate . stash , false , invalidityString ) ; return false ; } db . setIdentityInvalidity ( candidate . stash , true ) ; return true ; } ; export const checkOffline = async ( db , candidate ) => { const totalOffline = candidate . offlineAccumulated / WEEK ; if ( totalOffline > 0.02 ) { db . setOfflineAccumulatedInvalidity ( candidate . stash , false ) ; return false ; } else { db . setOfflineAccumulatedInvalidity ( candidate . stash , true ) ; return true ; } } ; export const checkRewardDestination = async ( db , chaindata , candidate ) => { const isStaked = await chaindata . destinationIsStaked ( candidate . stash ) ; if ( ! isStaked ) { await db . setRewardDestinationInvalidity ( candidate . stash , false ) ; return false ; } else { await db . setRewardDestinationInvalidity ( candidate . stash , true ) ; return true ; } } ; export const checkCommission = async ( db , chaindata , targetCommission , candidate ) => { const [ commission , err ] = await chaindata . getCommission ( candidate . stash ) ; if ( err ) { logger . warn ( `{CheckComssion} there was an error: ${ err }` ) ; return false ; } if ( commission > targetCommission ) { const invalidityString = `${ candidate . name } commission is set higher than the maximum allowed. Set: ${ commission / Math . pow ( 10 , 7 ) }% Allowed: ${ targetCommission / Math . pow ( 10 , 7 ) }%` ; db . setCommissionInvalidity ( candidate . stash , false , invalidityString ) ; return false ; } else { db . setCommissionInvalidity ( candidate . stash , true ) ; return true ; } } ; export const checkSelfStake = async ( db , chaindata , targetSelfStake , candidate ) => { if ( ! candidate . skipSelfStake ) { const [ bondedAmt , err2 ] = await chaindata . getBondedAmount ( candidate . stash ) ; let invalidityString ; if ( err2 ) { invalidityString = `${ candidate . name } ${ err2 }` ; await db . setSelfStakeInvalidity ( candidate . stash , false , invalidityString ) ; return false ; } if ( bondedAmt < targetSelfStake ) { invalidityString = `${ candidate . name } has less than the minimum amount bonded: ${ bondedAmt } is bonded.` ; await db . setSelfStakeInvalidity ( candidate . stash , false , invalidityString ) ; return false ; } } await db . setSelfStakeInvalidity ( candidate . stash , true ) ; return true ; } ; export const checkUnclaimed = async ( db , chaindata , unclaimedEraThreshold , candidate ) => { const [ currentEra , err3 ] = await chaindata . getActiveEraIndex ( ) ; const threshold = currentEra - unclaimedEraThreshold - 1 ; if ( candidate . unclaimedEras && ! candidate . unclaimedEras . every ( ( era ) => era > threshold ) ) { const invalidityString = `${ candidate . name } has unclaimed eras: ${ candidate . unclaimedEras } prior to era: ${ threshold + 1 }` ; await db . setUnclaimedInvalidity ( candidate . stash , false , invalidityString ) ; return false ; } else { await db . setUnclaimedInvalidity ( candidate . stash , true ) ; return true ; } } ; export const checkBlocked = async ( db , chaindata , candidate ) => { const isBlocked = await chaindata . getBlocked ( candidate . stash ) ; if ( isBlocked ) { const invalidityString = `${ candidate . name } blocks external nominations` ; await db . setBlockedInvalidity ( candidate . stash , false , invalidityString ) ; return false ; } else { await db . setBlockedInvalidity ( candidate . stash , true ) ; return true ; } } ; export const checkKusamaRank = async ( db , candidate ) => { try { if ( ! ! candidate . kusamaStash ) { const url = `${ KOTVBackendEndpoint }/candidate/${ candidate . kusamaStash }` ; const res = await axios . get ( url ) ; if ( ! ! res . data . invalidityReasons ) { const invalidityReason = `${ candidate . name } has a kusama node that is invalid: ${ res . data . invalidityReasons }` ; await db . setKusamaRankInvalidity ( candidate . stash , false , invalidityReason ) ; return false ; } if ( Number ( res . data . rank ) < 25 ) { const invalidityReason = `${ candidate . name } has a Kusama stash with lower than 25 rank in the Kusama OTV programme: ${ res . data . rank }.` ; await db . setKusamaRankInvalidity ( candidate . stash , false , invalidityReason ) ; return false ; } } await db . setKusamaRankInvalidity ( candidate . stash , true ) ; return true ; } catch ( e ) { logger . info ( `Error trying to get kusama data...` ) ; } } ;