[catboost] better exception message and better debug experience for snapshot recovery;
[catboost] better exception message and better debug experience for snapshot recovery;
([arc::mergeid] 68481e81-c55b4e65-f4a39c46-2b6b470d)
#include <library/digest/crc32c/crc32c.h>
#include <library/digest/md5/md5.h>
-#include <util/generic/guid.h>
#include <util/folder/path.h>
-#include <util/system/fs.h>
+#include <util/generic/guid.h>
#include <util/stream/file.h>
+#include <util/stream/labeled.h>
+#include <util/system/fs.h>
TLearnContext::~TLearnContext() {
if (Params.SystemOptions->IsMaster()) {
@@ -167,14 +168,27 @@ bool TLearnContext::TryLoadProgress() {
try {
TProgressHelper(ToString(ETaskType::CPU)).CheckedLoad(Files.SnapshotFile, [&](TIFStream* in)
{
- TLearnProgress learnProgressRestored = LearnProgress; // use progress copy to avoid partial deserialization of corrupted progress file
+ // use progress copy to avoid partial deserialization of corrupted progress file
+ TLearnProgress learnProgressRestored = LearnProgress;
TProfileInfoData ProfileRestored;
- ::LoadMany(in, Rand, learnProgressRestored, ProfileRestored); // fail here does nothing with real LearnProgress
+
+ // fail here does nothing with real LearnProgress
+ ::LoadMany(in, Rand, learnProgressRestored, ProfileRestored);
+
const bool paramsCompatible = NCatboostOptions::IsParamsCompatible(
- learnProgressRestored.SerializedTrainParams, LearnProgress.SerializedTrainParams);
+ learnProgressRestored.SerializedTrainParams,
+ LearnProgress.SerializedTrainParams);
+ CATBOOST_DEBUG_LOG
+ << LabeledOutput(learnProgressRestored.SerializedTrainParams) << ' '
+ << LabeledOutput(LearnProgress.SerializedTrainParams) << Endl;
CB_ENSURE(paramsCompatible, "Saved model's params are different from current model's params");
+
const bool poolCompatible = (learnProgressRestored.PoolCheckSum == LearnProgress.PoolCheckSum);
- CB_ENSURE(poolCompatible, "Current pool differs from the original pool");
+ CB_ENSURE(
+ poolCompatible,
+ "Current pool differs from the original pool "
+ LabeledOutput(learnProgressRestored.PoolCheckSum, LearnProgress.PoolCheckSum));
+
LearnProgress = std::move(learnProgressRestored);
Profile.InitProfileInfo(std::move(ProfileRestored));
LearnProgress.SerializedTrainParams = ToString(Params); // substitute real
