pub(crate) async fn check_for_upgrade(&self) -> OrchestratorResult<()> {
        let latest_registry_version = self.registry.get_latest_version();
        // Determine the subnet_id using the local CUP.
        let latest_subnet_id = if let Some(cup_with_proto) = self.cup_provider.get_local_cup() {
            let cup = &cup_with_proto.cup;
            let subnet_id = get_subnet_id(&*self.registry.registry_client, cup).map_err(|err| {
                OrchestratorError::UpgradeError(format!(
                    "Couldn't extract the subnet id from the local CUP: {:?}",
                    err
                ))
            })?;
            if should_node_become_unassigned(
                &*self.registry.registry_client,
                self.node_id,
                subnet_id,
                cup,
            ) {
                return Err(OrchestratorError::UpgradeError(
                    "Unassignment is not implemented".to_string(),
                ));
            }
            subnet_id
        } else {
            // If the local CUP was not found we first try to create one from the registry.
            if let Ok(cup) = self.registry.get_registry_cup(latest_registry_version) {
                self.cup_provider
                    .persist_cup(&CUPWithOriginalProtobuf::from_cup(cup))?;
                return Ok(());
            }
            // If we couldn't create a registry CUP, this node is not assigned;
            // check if it needs to be upgraded.
            return self.check_for_upgrade_as_unassigned().await;
        };

        // Now we know the subnet_id and we're assigned; start the replica if necessary.
        if !self.replica_process.lock().unwrap().is_running() {
            return self.start_replica(&self.replica_version, latest_subnet_id);
        }

        // 0. Special case for when we are doing boostrap subnet recovery for
        // nns and replacing the local registry store. Because we replace the
        // contents of the local registry store in the process of doing this, we
        // will not perpetually hit this case, and thus it is not important to
        // check the height.
        if let Some(registry_store_uri) = self
            .registry
            .registry_client
            .get_cup_contents(latest_subnet_id, latest_registry_version)
            .ok()
            .and_then(|record| record.value.and_then(|v| v.registry_store_uri))
        {
            let cup = self
                .registry
                .get_registry_cup(latest_registry_version)
                .expect("A registry cup must be present in the the registry");

            self.cup_provider.persist_cup_deprecated(
                &CUPWithOriginalProtobuf::from_cup(cup),
                latest_subnet_id,
            )?;

            warn!(
                self.logger,
                "Downloading registry data from {} with hash {} for subnet recovery",
                registry_store_uri.uri,
                registry_store_uri.hash,
            );

            let downloader = FileDownloader::new(Some(self.logger.clone()));
            let local_store_location = tempfile::tempdir()
                .expect("temporary location for local store download could not be created")
                .into_path();
            std::fs::create_dir_all(&local_store_location).expect("Directory should be created");

            if let Err(e) = downloader
                .download_and_extract_tar_gz(
                    &registry_store_uri.uri,
                    &local_store_location,
                    Some(registry_store_uri.hash),
                )
                .await
            {
                warn!(
                    self.logger,
                    "Download of registry store for nns subnet recovery failed {}", e,
                );
                return Err(OrchestratorError::FileDownloadError(e));
            }

            if let Err(e) = self.stop_replica() {
                // Even though we fail to stop the replica, we should still
                // replace the registry local store, so we simply issue a warning.
                warn!(self.logger, "Failed to stop replica with error {:?}", e);
            }

            let new_local_store = LocalStoreImpl::new(local_store_location);
            self.nns_registry_replicator
                .stop_polling_and_set_local_registry_data(&new_local_store);

            utils::reexec_current_process(&self.logger);
        }

        // Get latest CUP from peers & local CUPs on disk (fallback to registry)
        let cup = self.cup_provider.get_latest_cup(latest_subnet_id).await?;

        let cup_registry_version = cup.cup.content.registry_version();

        let subnet_record = self
            .registry
            .get_subnet_record(latest_subnet_id, cup_registry_version)?;

        // Determine version of release in the latest registry version.
        let latest_replica_version =
            RegistryHelper::get_replica_version_from_subnet_record(subnet_record)?;

        // If we are already running what is the latest version in the registry,
        // there cannot be an upgrade. No need to check for CUPs, simply return.
        // The exception to this is when we have a recovery CUP (which we can
        // detect by seeing that the public key of the subnet has changed).

        // Note that we do not allow version transitions v1 -> v2 -> v1.
        // Even without this optimization, the orchestrator would not trigger restarting
        // the replica (e.g. in cases we directly upgrade v1 -> v1).
        if latest_replica_version == self.replica_version {
            debug!(
                self.logger,
                "Latest version from registry {} - already running, no CUP fetching",
                latest_replica_version
            );
            return Ok(());
        }

        info!(
            self.logger,
            "Runnig upgrade loop. Latest IC version in registry: {}, running IC version {:?}",
            latest_replica_version,
            self.replica_version.as_ref(),
        );

        // Attempt to pro-actively download the replica version for
        // the highest current registry version. Note that the current
        // subnetwork might not yet have executed that upgrade (in
        // which case the actual replica version to boot will be downloaded later)

        // We download that version, but don't actually do anything with it.
        // The code will not re-download a replica version that we have previously
        // downloaded.
        //
        // Do not abort here, since it is possible we want to upgrade to v2, but we try
        // to download v3. If download of v3 fails, we should still continue because
        // upgrade to v2 might succeed.
        //
        // We delete previously downloaded release packages, as we cannot be sure that
        // their content is correct (e.g. when redeploying on the same machine).
        let _ = self
            .release_package_provider
            .download_release_package(latest_replica_version.clone())
            .await;

        // 2. Check for upgrade based on the registry verison used in current subnet

        // For the CUP's registry version, get replica version of current subnet.
        let subnet_record = self
            .registry
            .get_subnet_record(latest_subnet_id, cup_registry_version)?;
        let new_replica_version =
            &RegistryHelper::get_replica_version_from_subnet_record(subnet_record)?;

        // Version is identical, no upgrade needed
        if new_replica_version == &self.replica_version {
            info!(
                self.logger,
                "Expecting upgrade to version {}, but highest CUP has version {}, running {}",
                latest_replica_version,
                new_replica_version,
                self.replica_version
            );
            return Ok(());
        }

        // Now that we know we are upgrading, persist the CUP.
        self.cup_provider
            .persist_cup_deprecated(&cup, latest_subnet_id)?;

        info!(
            self.logger,
            "Replica upgrade detected: old version {:?} -> new version {}",
            self.replica_version,
            new_replica_version
        );

        self.download_and_upgrade(new_replica_version).await
    }
/// Checks for a new release package, and if found, upgrades to this release
/// package
Checks for a new release package, and if found, upgrades to this release
package
Checks for a new release package and if found upgrades to this release package
pub(crate) async fn check_for_upgrade(&self) -> OrchestratorResult<()> {
        let latest_registry_version = self.registry.get_latest_version();
        // Determine the subnet_id using the local CUP.
        let latest_subnet_id = if let Some(cup_with_proto) = self.cup_provider.get_local_cup() {
            let cup = &cup_with_proto.cup;
            let subnet_id = get_subnet_id(&*self.registry.registry_client, cup).map_err(|err| {
                OrchestratorError::UpgradeError(format!(
                    "Couldn't extract the subnet id from the local CUP: {:?}",
                    err
                ))
            })?;
            if should_node_become_unassigned(
                &*self.registry.registry_client,
                self.node_id,
                subnet_id,
                cup,
            ) {
                return Err(OrchestratorError::UpgradeError(
                    "Unassignment is not implemented".to_string(),
                ));
            }
            subnet_id
        } else {
            // If the local CUP was not found we first try to create one from the registry.
            if let Ok(cup) = self.registry.get_registry_cup(latest_registry_version) {
                self.cup_provider
                    .persist_cup(&CUPWithOriginalProtobuf::from_cup(cup))?;
                return Ok(());
            }
            // If we couldn't create a registry CUP, this node is not assigned;
            // check if it needs to be upgraded.
            return self.check_for_upgrade_as_unassigned().await;
        };
        // Now we know the subnet_id and we're assigned; start the replica if necessary.
        if !self.replica_process.lock().unwrap().is_running() {
            return self.start_replica(&self.replica_version, latest_subnet_id);
        }
        // 0. Special case for when we are doing boostrap subnet recovery for
        // nns and replacing the local registry store. Because we replace the
        // contents of the local registry store in the process of doing this, we
        // will not perpetually hit this case, and thus it is not important to
        // check the height.
        if let Some(registry_store_uri) = self
            .registry
            .registry_client
            .get_cup_contents(latest_subnet_id, latest_registry_version)
            .ok()
            .and_then(|record| record.value.and_then(|v| v.registry_store_uri))
        {
            let cup = self
                .registry
                .get_registry_cup(latest_registry_version)
                .expect("A registry cup must be present in the the registry");
            self.cup_provider.persist_cup_deprecated(
                &CUPWithOriginalProtobuf::from_cup(cup),
                latest_subnet_id,
            )?;
            warn!(
                self.logger,
                "Downloading registry data from {} with hash {} for subnet recovery",
                registry_store_uri.uri,
                registry_store_uri.hash,
            );
            let downloader = FileDownloader::new(Some(self.logger.clone()));
            let local_store_location = tempfile::tempdir()
                .expect("temporary location for local store download could not be created")
                .into_path();
            std::fs::create_dir_all(&local_store_location).expect("Directory should be created");
            if let Err(e) = downloader
                .download_and_extract_tar_gz(
                    &registry_store_uri.uri,
                    &local_store_location,
                    Some(registry_store_uri.hash),
                )
                .await
            {
                warn!(
                    self.logger,
                    "Download of registry store for nns subnet recovery failed {}", e,
                );
                return Err(OrchestratorError::FileDownloadError(e));
            }
            if let Err(e) = self.stop_replica() {
                // Even though we fail to stop the replica, we should still
                // replace the registry local store, so we simply issue a warning.
                warn!(self.logger, "Failed to stop replica with error {:?}", e);
            }
            let new_local_store = LocalStoreImpl::new(local_store_location);
            self.nns_registry_replicator
                .stop_polling_and_set_local_registry_data(&new_local_store);
            utils::reexec_current_process(&self.logger);
        }
        // Get latest CUP from peers & local CUPs on disk (fallback to registry)
        let cup = self.cup_provider.get_latest_cup(latest_subnet_id).await?;
        let cup_registry_version = cup.cup.content.registry_version();
        let subnet_record = self
            .registry
            .get_subnet_record(latest_subnet_id, cup_registry_version)?;
        // Determine version of release in the latest registry version.
        let latest_replica_version =
            RegistryHelper::get_replica_version_from_subnet_record(subnet_record)?;
        // If we are already running what is the latest version in the registry,
        // there cannot be an upgrade. No need to check for CUPs, simply return.
        // The exception to this is when we have a recovery CUP (which we can
        // detect by seeing that the public key of the subnet has changed).
        // Note that we do not allow version transitions v1 -> v2 -> v1.
        // Even without this optimization, the orchestrator would not trigger restarting
        // the replica (e.g. in cases we directly upgrade v1 -> v1).
        if latest_replica_version == self.replica_version {
            debug!(
                self.logger,
                "Latest version from registry {} - already running, no CUP fetching",
                latest_replica_version
            );
            return Ok(());
        }
        info!(
            self.logger,
            "Runnig upgrade loop. Latest IC version in registry: {}, running IC version {:?}",
            latest_replica_version,
            self.replica_version.as_ref(),
        );
        // Attempt to pro-actively download the replica version for
        // the highest current registry version. Note that the current
        // subnetwork might not yet have executed that upgrade (in
        // which case the actual replica version to boot will be downloaded later)
        // We download that version, but don't actually do anything with it.
        // The code will not re-download a replica version that we have previously
        // downloaded.
        //
        // Do not abort here, since it is possible we want to upgrade to v2, but we try
        // to download v3. If download of v3 fails, we should still continue because
        // upgrade to v2 might succeed.
        //
        // We delete previously downloaded release packages, as we cannot be sure that
        // their content is correct (e.g. when redeploying on the same machine).
        let _ = self
            .release_package_provider
            .download_release_package(latest_replica_version.clone())
            .await;
        // 2. Check for upgrade based on the registry verison used in current subnet
        // For the CUP's registry version, get replica version of current subnet.
        let subnet_record = self
            .registry
            .get_subnet_record(latest_subnet_id, cup_registry_version)?;
        let new_replica_version =
            &RegistryHelper::get_replica_version_from_subnet_record(subnet_record)?;
        // Version is identical, no upgrade needed
        if new_replica_version == &self.replica_version {
            info!(
                self.logger,
                "Expecting upgrade to version {}, but highest CUP has version {}, running {}",
                latest_replica_version,
                new_replica_version,
                self.replica_version
            );
            return Ok(());
        }
        // Now that we know we are upgrading, persist the CUP.
        self.cup_provider
            .persist_cup_deprecated(&cup, latest_subnet_id)?;
        info!(
            self.logger,
            "Replica upgrade detected: old version {:?} -> new version {}",
            self.replica_version,
            new_replica_version
        );
        self.download_and_upgrade(new_replica_version).await
    }
pub ( crate ) async fn check_for_upgrade ( & self ) -> OrchestratorResult < ( ) > { let latest_registry_version = self . registry . get_latest_version ( ) ; // Determine the subnet_id using the local CUP. let latest_subnet_id = if let Some ( cup_with_proto ) = self . cup_provider . get_local_cup ( ) { let cup = & cup_with_proto . cup ; let subnet_id = get_subnet_id ( & * self . registry . registry_client , cup ) . map_err ( | err | { OrchestratorError :: UpgradeError ( format ! ( " " err ) ) } ) ? ; if should_node_become_unassigned ( & * self . registry . registry_client , self . node_id , subnet_id , cup , ) { return Err ( OrchestratorError :: UpgradeError ( " " . to_string ( ) , ) ) ; } subnet_id } else { // If the local CUP was not found we first try to create one from the registry. if let Ok ( cup ) = self . registry . get_registry_cup ( latest_registry_version ) { self . cup_provider . persist_cup ( & CUPWithOriginalProtobuf :: from_cup ( cup ) ) ? ; return Ok ( ( ) ) ; } // If we couldn't create a registry CUP, this node is not assigned; // check if it needs to be upgraded. return self . check_for_upgrade_as_unassigned ( ) . await ; } ; // Now we know the subnet_id and we're assigned; start the replica if necessary. if ! self . replica_process . lock ( ) . unwrap ( ) . is_running ( ) { return self . start_replica ( & self . replica_version , latest_subnet_id ) ; } // 0. Special case for when we are doing boostrap subnet recovery for // nns and replacing the local registry store. Because we replace the // contents of the local registry store in the process of doing this, we // will not perpetually hit this case, and thus it is not important to // check the height. if let Some ( registry_store_uri ) = self . registry . registry_client . get_cup_contents ( latest_subnet_id , latest_registry_version ) . ok ( ) . and_then ( | record | record . value . and_then ( | v | v . registry_store_uri ) ) { let cup = self . registry . get_registry_cup ( latest_registry_version ) . expect ( " " ) ; self . cup_provider . persist_cup_deprecated ( & CUPWithOriginalProtobuf :: from_cup ( cup ) , latest_subnet_id , ) ? ; warn ! ( self logger " " registry_store_uri uri registry_store_uri hash ) ; let downloader = FileDownloader :: new ( Some ( self . logger . clone ( ) ) ) ; let local_store_location = tempfile :: tempdir ( ) . expect ( " " ) . into_path ( ) ; std :: fs :: create_dir_all ( & local_store_location ) . expect ( " " ) ; if let Err ( e ) = downloader . download_and_extract_tar_gz ( & registry_store_uri . uri , & local_store_location , Some ( registry_store_uri . hash ) , ) . await { warn ! ( self logger " " e ) ; return Err ( OrchestratorError :: FileDownloadError ( e ) ) ; } if let Err ( e ) = self . stop_replica ( ) { // Even though we fail to stop the replica, we should still // replace the registry local store, so we simply issue a warning. warn ! ( self logger " " e ) ; } let new_local_store = LocalStoreImpl :: new ( local_store_location ) ; self . nns_registry_replicator . stop_polling_and_set_local_registry_data ( & new_local_store ) ; utils :: reexec_current_process ( & self . logger ) ; } // Get latest CUP from peers & local CUPs on disk (fallback to registry) let cup = self . cup_provider . get_latest_cup ( latest_subnet_id ) . await ? ; let cup_registry_version = cup . cup . content . registry_version ( ) ; let subnet_record = self . registry . get_subnet_record ( latest_subnet_id , cup_registry_version ) ? ; // Determine version of release in the latest registry version. let latest_replica_version = RegistryHelper :: get_replica_version_from_subnet_record ( subnet_record ) ? ; // If we are already running what is the latest version in the registry, // there cannot be an upgrade. No need to check for CUPs, simply return. // The exception to this is when we have a recovery CUP (which we can // detect by seeing that the public key of the subnet has changed). // Note that we do not allow version transitions v1 -> v2 -> v1. // Even without this optimization, the orchestrator would not trigger restarting // the replica (e.g. in cases we directly upgrade v1 -> v1). if latest_replica_version == self . replica_version { debug ! ( self logger " " latest_replica_version ) ; return Ok ( ( ) ) ; } info ! ( self logger " " latest_replica_version self replica_version as_ref ( ) ) ; // Attempt to pro-actively download the replica version for // the highest current registry version. Note that the current // subnetwork might not yet have executed that upgrade (in // which case the actual replica version to boot will be downloaded later) // We download that version, but don't actually do anything with it. // The code will not re-download a replica version that we have previously // downloaded. // // Do not abort here, since it is possible we want to upgrade to v2, but we try // to download v3. If download of v3 fails, we should still continue because // upgrade to v2 might succeed. // // We delete previously downloaded release packages, as we cannot be sure that // their content is correct (e.g. when redeploying on the same machine). let _ = self . release_package_provider . download_release_package ( latest_replica_version . clone ( ) ) . await ; // 2. Check for upgrade based on the registry verison used in current subnet // For the CUP's registry version, get replica version of current subnet. let subnet_record = self . registry . get_subnet_record ( latest_subnet_id , cup_registry_version ) ? ; let new_replica_version = & RegistryHelper :: get_replica_version_from_subnet_record ( subnet_record ) ? ; // Version is identical, no upgrade needed if new_replica_version == & self . replica_version { info ! ( self logger " " latest_replica_version new_replica_version self replica_version ) ; return Ok ( ( ) ) ; } // Now that we know we are upgrading, persist the CUP. self . cup_provider . persist_cup_deprecated ( & cup , latest_subnet_id ) ? ; info ! ( self logger " " self replica_version new_replica_version ) ; self . download_and_upgrade ( new_replica_version ) . await }
Checks for a new release package, and if found, upgrades to this release
package
Checks for a new release package and if found upgrades to this release package
&self
&self