Finish the alpha of IndexDownloader
MODIFY
MagicalCryptoWallet.Backend/Controllers/BlockchainController.cs
MagicalCryptoWallet.Backend/Controllers/BlockchainController.cs
@@ -201,10 +201,12 @@ namespace MagicalCryptoWallet.Backend.Controllers
/// <param name="bestKnownBlockHash">The best block hash the client knows its filter.</param>
/// <returns>An array of block hash : element count : filter pairs.</returns>
/// <response code="200">An array of block hash : element count : filter pairs.</response>
+ /// <response code="204">When the provided hash is the tip.</response>
/// <response code="400">The provided hash was malformed.</response>
/// <response code="404">If the hash is not found. This happens at blockhain reorg.</response>
[HttpGet("filters/{bestKnownBlockHash}")]
[ProducesResponseType(200)] // Note: If you add typeof(IList<string>) then swagger UI visualization will be ugly.
+ [ProducesResponseType(200)]
[ProducesResponseType(400)]
[ProducesResponseType(404)]
public IActionResult GetFilters(string bestKnownBlockHash)
@@ -216,11 +218,16 @@ namespace MagicalCryptoWallet.Backend.Controllers
var knownHash = new uint256(bestKnownBlockHash);
- IEnumerable<string> filters = Global.IndexBuilderService.GetFilters(knownHash);
+ IEnumerable<string> filters = Global.IndexBuilderService.GetFilters(knownHash, out bool found);
+
+ if(!found)
+ {
+ return NotFound($"Provided {nameof(bestKnownBlockHash)} is not found: {bestKnownBlockHash}.");
+ }
if(filters.Count() == 0)
{
- return NotFound($"Provided {nameof(bestKnownBlockHash)} is not found: {bestKnownBlockHash} or it is the tip.");
+ return NoContent();
}
return Ok(filters);

MODIFY
MagicalCryptoWallet.Tests/P2pTests.cs
MagicalCryptoWallet.Tests/P2pTests.cs
@@ -217,7 +217,7 @@ namespace MagicalCryptoWallet.Tests
// Test initial syncronization.
var times = 0;
uint256 firstHash = await rpc.GetBlockHashAsync(0);
- while (indexBuilderService.GetFilters(firstHash).Count() != 101)
+ while (indexBuilderService.GetFilters(firstHash, out bool found6).Count() != 101)
{
if (times > 500) // 30 sec
{
@@ -230,8 +230,9 @@ namespace MagicalCryptoWallet.Tests
// Test later syncronization.
regtestNode.Generate(10);
times = 0;
- while (indexBuilderService.GetFilters(firstHash).Count() != 111)
+ while (indexBuilderService.GetFilters(firstHash, out bool found5).Count() != 111)
{
+ Assert.True(found5);
if (times > 500) // 30 sec
{
throw new TimeoutException($"{nameof(IndexBuilderService)} test timed out.");
@@ -242,12 +243,17 @@ namespace MagicalCryptoWallet.Tests
// Test correct number of filters is received.
var hundredthHash = await rpc.GetBlockHashAsync(100);
- Assert.Equal(11, indexBuilderService.GetFilters(hundredthHash).Count());
+ Assert.Equal(11, indexBuilderService.GetFilters(hundredthHash, out bool found).Count());
+ Assert.True(found);
var bestHash = await rpc.GetBestBlockHashAsync();
- Assert.Empty(indexBuilderService.GetFilters(bestHash));
+ Assert.Empty(indexBuilderService.GetFilters(bestHash, out bool found2));
+ Assert.True(found2);
+ Assert.Empty(indexBuilderService.GetFilters(uint256.Zero, out bool found3));
+ Assert.False(found3);
// Test filter block hashes are correct.
- var filters = indexBuilderService.GetFilters(firstHash).ToArray();
+ var filters = indexBuilderService.GetFilters(firstHash, out bool found4).ToArray();
+ Assert.True(found4);
for (int i = 0; i < 111; i++)
{
var expectedHash = await rpc.GetBlockHashAsync(i + 1);

MODIFY
MagicalCryptoWallet/Services/IndexBuilderService.cs
MagicalCryptoWallet/Services/IndexBuilderService.cs
@@ -309,16 +309,17 @@ namespace MagicalCryptoWallet.Services
});
}
- public IEnumerable<string> GetFilters(uint256 bestKnownBlockHash)
+ public IEnumerable<string> GetFilters(uint256 bestKnownBlockHash, out bool found)
{
using (IndexLock.Lock())
{
- var found = false;
+ found = false;
+ var filters = new List<string>();
foreach (var filter in Index)
{
if (found)
{
- yield return filter.ToLine();
+ filters.Add(filter.ToLine());
}
else
{
@@ -328,6 +329,8 @@ namespace MagicalCryptoWallet.Services
}
}
}
+
+ return filters;
}
}

MODIFY
MagicalCryptoWallet/Services/IndexDownloader.cs
MagicalCryptoWallet/Services/IndexDownloader.cs
@@ -10,6 +10,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
+using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
@@ -98,120 +99,66 @@ namespace MagicalCryptoWallet.Services
// If stop was requested return.
if (IsRunning == false) return;
- //int height = StartingHeight.Value;
- //uint256 prevHash = null;
- //using (await IndexLock.LockAsync())
- //{
- // if (Index.Count != 0)
- // {
- // height = Index.Last().BlockHeight.Value + 1;
- // prevHash = Index.Last().BlockHash;
- // }
- //}
-
- //Block block = null;
- //try
- //{
- // block = await RpcClient.GetBlockAsync(height);
- //}
- //catch (RPCException) // if the block didn't come yet
- //{
- // // ToDO: If this happens, we should do `waitforblock` RPC instead of periodically asking.
- // // In that case we must also make sure the correct error message comes.
- // await Task.Delay(1000);
- // continue;
- //}
-
- //if (prevHash != null)
- //{
- // // In case of reorg:
- // if (prevHash != block.Header.HashPrevBlock)
- // {
- // Logger.LogInfo<IndexBuilderService>($"REORG Invalid Block: {prevHash}");
- // // 1. Rollback index
- // using (await IndexLock.LockAsync())
- // {
- // Index.RemoveAt(Index.Count - 1);
- // }
-
- // // 2. Serialize Index. (Remove last line.)
- // var lines = File.ReadAllLines(IndexFilePath);
- // File.WriteAllLines(IndexFilePath, lines.Take(lines.Length - 1).ToArray());
-
- // // 3. Rollback Bech32UtxoSet
- // Bech32UtxoSetHistory.Rollback(Bech32UtxoSet); // The Bech32UtxoSet MUST be recovered to its previous state.
-
- // // 4. Serialize Bech32UtxoSet.
- // await File.WriteAllLinesAsync(Bech32UtxoSetFilePath, Bech32UtxoSet
- // .Select(entry => entry.Key.Hash + ":" + entry.Key.N + ":" + ByteHelpers.ToHex(entry.Value.ToCompressedBytes())));
-
- // // 5. Skip the current block.
- // continue;
- // }
- //}
-
- //Bech32UtxoSetHistory.ClearActionHistory(); //reset history.
-
- //var scripts = new HashSet<Script>();
-
- //foreach (var tx in block.Transactions)
- //{
- // for (int i = 0; i < tx.Outputs.Count; i++)
- // {
- // var output = tx.Outputs[i];
- // if (!output.ScriptPubKey.IsPayToScriptHash && output.ScriptPubKey.IsWitness)
- // {
- // var outpoint = new OutPoint(tx.GetHash(), i);
- // Bech32UtxoSet.Add(outpoint, output.ScriptPubKey);
- // Bech32UtxoSetHistory.StoreAction(ActionHistoryHelper.Operation.Add, outpoint, output.ScriptPubKey);
- // scripts.Add(output.ScriptPubKey);
- // }
- // }
-
- // foreach (var input in tx.Inputs)
- // {
- // var found = Bech32UtxoSet.SingleOrDefault(x => x.Key == input.PrevOut);
- // if (found.Key != default)
- // {
- // Script val = Bech32UtxoSet[input.PrevOut];
- // Bech32UtxoSet.Remove(input.PrevOut);
- // Bech32UtxoSetHistory.StoreAction(ActionHistoryHelper.Operation.Remove, input.PrevOut, val);
- // scripts.Add(found.Value);
- // }
- // }
- //}
-
- //// https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki
- //// The parameter k MUST be set to the first 16 bytes of the hash of the block for which the filter
- //// is constructed.This ensures the key is deterministic while still varying from block to block.
- //var key = block.GetHash().ToBytes().Take(16).ToArray();
-
- //GolombRiceFilter filter = null;
- //if (scripts.Count != 0)
- //{
- // filter = GolombRiceFilter.Build(key, scripts.Select(x => x.ToCompressedBytes()));
- //}
-
- //var filterModel = new FilterModel
- //{
- // BlockHash = block.GetHash(),
- // BlockHeight = new Height(height),
- // Filter = filter
- //};
-
- //await File.AppendAllLinesAsync(IndexFilePath, new[] { filterModel.ToLine() });
- //using (await IndexLock.LockAsync())
- //{
- // Index.Add(filterModel);
- //}
- //if (File.Exists(Bech32UtxoSetFilePath))
- //{
- // File.Delete(Bech32UtxoSetFilePath);
- //}
- //await File.WriteAllLinesAsync(Bech32UtxoSetFilePath, Bech32UtxoSet
- // .Select(entry => entry.Key.Hash + ":" + entry.Key.N + ":" + ByteHelpers.ToHex(entry.Value.ToCompressedBytes())));
-
- //Logger.LogInfo<IndexBuilderService>($"Created filter for block: {height}.");
+ FilterModel bestKnownFilter;
+ using (await IndexLock.LockAsync())
+ {
+ if(Index.Count == 0)
+ {
+ bestKnownFilter = StartingFilter;
+ }
+ else
+ {
+ bestKnownFilter = Index.Last();
+ }
+ }
+
+ var response = await Client.SendAsync(HttpMethod.Get, $"/api/v1/btc/Blockchain/filters/{bestKnownFilter.BlockHash}");
+
+ if(response.StatusCode == HttpStatusCode.NoContent)
+ {
+ continue;
+ }
+ if(response.StatusCode == HttpStatusCode.OK)
+ {
+ var filters = await response.Content.ReadAsJsonAsync<List<string>>();
+
+ for(int i = 0; i < filters.Count; i++)
+ {
+ var filterModel = FilterModel.FromLine(filters[i], new Height(bestKnownFilter.BlockHeight.Value + i + 1));
+
+ await File.AppendAllLinesAsync(IndexFilePath, new[] { filterModel.ToLine() });
+ using (await IndexLock.LockAsync())
+ {
+ Index.Add(filterModel);
+ }
+ }
+
+ Logger.LogInfo<IndexDownloader>($"Downloaded filters for blocks from {bestKnownFilter.BlockHeight} to {Index.Last().BlockHeight}.");
+
+ continue;
+ }
+ else if(response.StatusCode == HttpStatusCode.NotFound)
+ {
+ // Reorg happened
+ Logger.LogInfo<IndexDownloader>($"REORG Invalid Block: {bestKnownFilter.BlockHash}");
+ // 1. Rollback index
+ using (await IndexLock.LockAsync())
+ {
+ Index.RemoveAt(Index.Count - 1);
+ }
+
+ // 2. Serialize Index. (Remove last line.)
+ var lines = File.ReadAllLines(IndexFilePath);
+ File.WriteAllLines(IndexFilePath, lines.Take(lines.Length - 1).ToArray());
+
+ // 3. Skip the last valid block.
+ continue;
+ }
+ else
+ {
+ var error = await response.Content.ReadAsStringAsync();
+ throw new HttpRequestException($"{response.StatusCode.ToReasonString()}: {error}");
+ }
}
catch (Exception ex)
{

Finish the alpha of IndexDownloader