private async Task<List<string>> GetSerializedUnoptimisedKeysParallel(string start = "init") {
ConcurrentDictionary<string, string> pairs = [];
var unoptimisedKeys = (await storageProvider.GetAllKeysAsync()).Where(static x => !x.Contains('/')).ToFrozenSet();
await Parallel.ForEachAsync(unoptimisedKeys, async (key, _) => {
try {
var data = await storageProvider.LoadObjectAsync<SyncResponse>(key, SyncResponseSerializerContext.Default.SyncResponse);
if (data is null) return;
pairs.TryAdd(key, data.NextBatch);
}
catch (Exception e) {
Console.WriteLine($"Failed to read {key}:");
throw;
}
});
var serializedKeys = new List<string>();
var currentKey = start;
while (pairs.TryGetValue(currentKey, out var nextKey)) {
serializedKeys.Add(currentKey);
currentKey = nextKey;
}
return serializedKeys;
}
// Usage:
List<string> serialisedKeys = await GetSerializedUnoptimisedKeysParallel();
var chunkSize = serialisedKeys.Count / Environment.ProcessorCount;
var chunks = serialisedKeys.Chunk(chunkSize+1).Select(x => (x.First(), x.Length)).ToList();
Console.WriteLine($"Got {chunks.Count} chunks:");
foreach (var chunk in chunks) {
Console.WriteLine($"Chunk {chunk.Item1} with length {chunk.Length}");
}
