api.go
func RecoverCluster(conf *Config, fsm FSM, logs LogStore, stable StableStore,
	snaps SnapshotStore, trans Transport, configuration Configuration) error {
	// Validate the Raft server config.
	if err := ValidateConfig(conf); err != nil {
		return err
	}

	// Sanity check the Raft peer configuration.
	if err := checkConfiguration(configuration); err != nil {
		return err
	}

	// Refuse to recover if there's no existing state. This would be safe to
	// do, but it is likely an indication of an operator error where they
	// expect data to be there and it's not. By refusing, we force them
	// to show intent to start a cluster fresh by explicitly doing a
	// bootstrap, rather than quietly fire up a fresh cluster here.
	hasState, err := HasExistingState(logs, stable, snaps)
	if err != nil {
		return fmt.Errorf("failed to check for existing state: %v", err)
	}
	if !hasState {
		return fmt.Errorf("refused to recover cluster with no initial state, this is probably an operator error")
	}

	// Attempt to restore any snapshots we find, newest to oldest.
	var snapshotIndex uint64
	var snapshotTerm uint64
	snapshots, err := snaps.List()
	if err != nil {
		return fmt.Errorf("failed to list snapshots: %v", err)
	}
	for _, snapshot := range snapshots {
		_, source, err := snaps.Open(snapshot.ID)
		if err != nil {
			// Skip this one and try the next. We will detect if we
			// couldn't open any snapshots.
			continue
		}
		defer source.Close()

		if err := fsm.Restore(source); err != nil {
			// Same here, skip and try the next one.
			continue
		}

		snapshotIndex = snapshot.Index
		snapshotTerm = snapshot.Term
		break
	}
	if len(snapshots) > 0 && (snapshotIndex == 0 || snapshotTerm == 0) {
		return fmt.Errorf("failed to restore any of the available snapshots")
	}

	// The snapshot information is the best known end point for the data
	// until we play back the Raft log entries.
	lastIndex := snapshotIndex
	lastTerm := snapshotTerm

	// Apply any Raft log entries past the snapshot.
	lastLogIndex, err := logs.LastIndex()
	if err != nil {
		return fmt.Errorf("failed to find last log: %v", err)
	}
	for index := snapshotIndex + 1; index <= lastLogIndex; index++ {
		var entry Log
		if err := logs.GetLog(index, &entry); err != nil {
			return fmt.Errorf("failed to get log at index %d: %v", index, err)
		}
		if entry.Type == LogCommand {
			_ = fsm.Apply(&entry)
		}
		lastIndex = entry.Index
		lastTerm = entry.Term
	}

	// Create a new snapshot, placing the configuration in as if it was
	// committed at index 1.
	snapshot, err := fsm.Snapshot()
	if err != nil {
		return fmt.Errorf("failed to snapshot FSM: %v", err)
	}
	version := getSnapshotVersion(conf.ProtocolVersion)
	sink, err := snaps.Create(version, lastIndex, lastTerm, configuration, 1, trans)
	if err != nil {
		return fmt.Errorf("failed to create snapshot: %v", err)
	}
	if err := snapshot.Persist(sink); err != nil {
		return fmt.Errorf("failed to persist snapshot: %v", err)
	}
	if err := sink.Close(); err != nil {
		return fmt.Errorf("failed to finalize snapshot: %v", err)
	}

	// Compact the log so that we don't get bad interference from any
	// configuration change log entries that might be there.
	firstLogIndex, err := logs.FirstIndex()
	if err != nil {
		return fmt.Errorf("failed to get first log index: %v", err)
	}
	if err := logs.DeleteRange(firstLogIndex, lastLogIndex); err != nil {
		return fmt.Errorf("log compaction failed: %v", err)
	}

	return nil
}
func RecoverCluster(conf *Config, fsm FSM, logs LogStore, stable StableStore,
	snaps SnapshotStore, trans Transport, configuration Configuration) error {
	// Validate the Raft server config.
	if err := ValidateConfig(conf); err != nil {
		return err
	}

	// Sanity check the Raft peer configuration.
	if err := checkConfiguration(configuration); err != nil {
		return err
	}

	// Refuse to recover if there's no existing state. This would be safe to
	// do, but it is likely an indication of an operator error where they
	// expect data to be there and it's not. By refusing, we force them
	// to show intent to start a cluster fresh by explicitly doing a
	// bootstrap, rather than quietly fire up a fresh cluster here.
	hasState, err := HasExistingState(logs, stable, snaps)
	if err != nil {
		return fmt.Errorf("failed to check for existing state: %v", err)
	}
	if !hasState {
		return fmt.Errorf("refused to recover cluster with no initial state, this is probably an operator error")
	}

	// Attempt to restore any snapshots we find, newest to oldest.
	var snapshotIndex uint64
	var snapshotTerm uint64
	snapshots, err := snaps.List()
	if err != nil {
		return fmt.Errorf("failed to list snapshots: %v", err)
	}
	for _, snapshot := range snapshots {
		_, source, err := snaps.Open(snapshot.ID)
		if err != nil {
			// Skip this one and try the next. We will detect if we
			// couldn't open any snapshots.
			continue
		}
		defer source.Close()

		if err := fsm.Restore(source); err != nil {
			// Same here, skip and try the next one.
			continue
		}

		snapshotIndex = snapshot.Index
		snapshotTerm = snapshot.Term
		break
	}
	if len(snapshots) > 0 && (snapshotIndex == 0 || snapshotTerm == 0) {
		return fmt.Errorf("failed to restore any of the available snapshots")
	}

	// The snapshot information is the best known end point for the data
	// until we play back the Raft log entries.
	lastIndex := snapshotIndex
	lastTerm := snapshotTerm

	// Apply any Raft log entries past the snapshot.
	lastLogIndex, err := logs.LastIndex()
	if err != nil {
		return fmt.Errorf("failed to find last log: %v", err)
	}
	for index := snapshotIndex + 1; index <= lastLogIndex; index++ {
		var entry Log
		if err := logs.GetLog(index, &entry); err != nil {
			return fmt.Errorf("failed to get log at index %d: %v", index, err)
		}
		if entry.Type == LogCommand {
			_ = fsm.Apply(&entry)
		}
		lastIndex = entry.Index
		lastTerm = entry.Term
	}

	// Create a new snapshot, placing the configuration in as if it was
	// committed at index 1.
	snapshot, err := fsm.Snapshot()
	if err != nil {
		return fmt.Errorf("failed to snapshot FSM: %v", err)
	}
	version := getSnapshotVersion(conf.ProtocolVersion)
	sink, err := snaps.Create(version, lastIndex, lastTerm, configuration, 1, trans)
	if err != nil {
		return fmt.Errorf("failed to create snapshot: %v", err)
	}
	if err := snapshot.Persist(sink); err != nil {
		return fmt.Errorf("failed to persist snapshot: %v", err)
	}
	if err := sink.Close(); err != nil {
		return fmt.Errorf("failed to finalize snapshot: %v", err)
	}

	// Compact the log so that we don't get bad interference from any
	// configuration change log entries that might be there.
	firstLogIndex, err := logs.FirstIndex()
	if err != nil {
		return fmt.Errorf("failed to get first log index: %v", err)
	}
	if err := logs.DeleteRange(firstLogIndex, lastLogIndex); err != nil {
		return fmt.Errorf("log compaction failed: %v", err)
	}

	return nil
}
func RecoverCluster ( conf * Config , fsm FSM , logs LogStore , stable StableStore , snaps SnapshotStore , trans Transport , configuration Configuration ) error { // Validate the Raft server config. if err := ValidateConfig ( conf ) ; err != nil { return err } // Sanity check the Raft peer configuration. if err := checkConfiguration ( configuration ) ; err != nil { return err } // Refuse to recover if there's no existing state. This would be safe to // do, but it is likely an indication of an operator error where they // expect data to be there and it's not. By refusing, we force them // to show intent to start a cluster fresh by explicitly doing a // bootstrap, rather than quietly fire up a fresh cluster here. hasState , err := HasExistingState ( logs , stable , snaps ) if err != nil { return fmt . Errorf ( " " , err ) } if ! hasState { return fmt . Errorf ( " " ) } // Attempt to restore any snapshots we find, newest to oldest. var snapshotIndex uint64 var snapshotTerm uint64 snapshots , err := snaps . List ( ) if err != nil { return fmt . Errorf ( " " , err ) } for _ , snapshot := range snapshots { _ , source , err := snaps . Open ( snapshot . ID ) if err != nil { // Skip this one and try the next. We will detect if we // couldn't open any snapshots. continue } defer source . Close ( ) if err := fsm . Restore ( source ) ; err != nil { // Same here, skip and try the next one. continue } snapshotIndex = snapshot . Index snapshotTerm = snapshot . Term break } if len ( snapshots ) > 0 && ( snapshotIndex == 0 || snapshotTerm == 0 ) { return fmt . Errorf ( " " ) } // The snapshot information is the best known end point for the data // until we play back the Raft log entries. lastIndex := snapshotIndex lastTerm := snapshotTerm // Apply any Raft log entries past the snapshot. lastLogIndex , err := logs . LastIndex ( ) if err != nil { return fmt . Errorf ( " " , err ) } for index := snapshotIndex + 1 ; index <= lastLogIndex ; index ++ { var entry Log if err := logs . GetLog ( index , & entry ) ; err != nil { return fmt . Errorf ( " " , index , err ) } if entry . Type == LogCommand { _ = fsm . Apply ( & entry ) } lastIndex = entry . Index lastTerm = entry . Term } // Create a new snapshot, placing the configuration in as if it was // committed at index 1. snapshot , err := fsm . Snapshot ( ) if err != nil { return fmt . Errorf ( " " , err ) } version := getSnapshotVersion ( conf . ProtocolVersion ) sink , err := snaps . Create ( version , lastIndex , lastTerm , configuration , 1 , trans ) if err != nil { return fmt . Errorf ( " " , err ) } if err := snapshot . Persist ( sink ) ; err != nil { return fmt . Errorf ( " " , err ) } if err := sink . Close ( ) ; err != nil { return fmt . Errorf ( " " , err ) } // Compact the log so that we don't get bad interference from any // configuration change log entries that might be there. firstLogIndex , err := logs . FirstIndex ( ) if err != nil { return fmt . Errorf ( " " , err ) } if err := logs . DeleteRange ( firstLogIndex , lastLogIndex ) ; err != nil { return fmt . Errorf ( " " , err ) } return nil }
// RecoverCluster is used to manually force a new configuration in order to
// recover from a loss of quorum where the current configuration cannot be
// restored, such as when several servers die at the same time. This works by
// reading all the current state for this server, creating a snapshot with the
// supplied configuration, and then truncating the Raft log. This is the only
// safe way to force a given configuration without actually altering the log to
// insert any new entries, which could cause conflicts with other servers with
// different state.
//
// WARNING! This operation implicitly commits all entries in the Raft log, so
// in general this is an extremely unsafe operation. If you've lost your other
// servers and are performing a manual recovery, then you've also lost the
// commit information, so this is likely the best you can do, but you should be
// aware that calling this can cause Raft log entries that were in the process
// of being replicated but not yet be committed to be committed.
//
// Note the FSM passed here is used for the snapshot operations and will be
// left in a state that should not be used by the application. Be sure to
// discard this FSM and any associated state and provide a fresh one when
// calling NewRaft later.
//
// A typical way to recover the cluster is to shut down all servers and then
// run RecoverCluster on every server using an identical configuration. When
// the cluster is then restarted, and election should occur and then Raft will
// resume normal operation. If it's desired to make a particular server the
// leader, this can be used to inject a new configuration with that server as
// the sole voter, and then join up other new clean-state peer servers using
// the usual APIs in order to bring the cluster back into a known state.
RecoverCluster is used to manually force a new configuration in order to recover from a loss of quorum where the current configuration cannot be restored such as when several servers die at the same time . This works by reading all the current state for this server creating a snapshot with the supplied configuration and then truncating the Raft log . This is the only safe way to force a given configuration without actually altering the log to insert any new entries which could cause conflicts with other servers with different state . WARNING! This operation implicitly commits all entries in the Raft log so in general this is an extremely unsafe operation . If you ve lost your other servers and are performing a manual recovery then you ve also lost the commit information so this is likely the best you can do but you should be aware that calling this can cause Raft log entries that were in the process of being replicated but not yet be committed to be committed . Note the FSM passed here is used for the snapshot operations and will be left in a state that should not be used by the application . Be sure to discard this FSM and any associated state and provide a fresh one when calling NewRaft later . A typical way to recover the cluster is to shut down all servers and then run RecoverCluster on every server using an identical configuration . When the cluster is then restarted and election should occur and then Raft will resume normal operation . If it s desired to make a particular server the leader this can be used to inject a new configuration with that server as the sole voter and then join up other new clean - state peer servers using the usual APIs in order to bring the cluster back into a known state .