using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Auth; using Newtonsoft.Json.Linq; using NuGet.Services.Metadata.Catalog.Helpers; using NuGet.Services.Metadata.Catalog.Persistence; using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Threading.Tasks; namespace CatalogTests { class PartitioningTests { static void PrintMetrics(IDictionary> packages) { Console.WriteLine(); Console.WriteLine(packages.Count); int count = 0; foreach (var versions in packages.Values) { count += versions.Count; } Console.WriteLine(count); } static async Task> GetPopularPackages() { Uri reportUri = new Uri("https://nugetgallery.blob.core.windows.net/stats/popularity/recentpopularity.json"); HttpClient client = new HttpClient(); Console.WriteLine(reportUri); HttpResponseMessage reportResponse = await client.GetAsync(reportUri); string reportJson = await reportResponse.Content.ReadAsStringAsync(); JToken report = JToken.Parse(reportJson); HashSet packages = new HashSet(); foreach (JObject item in report) { string packageId = item["PackageId"].ToString().ToLowerInvariant(); packages.Add(packageId); } return packages; } public static void Test0() { Uri indexUri = new Uri("http://nugetprod0.blob.core.windows.net/ng-catalogs/0/index.json"); IDictionary> packages = IndexingHelpers.GetPackages(indexUri, true).Result; PrintMetrics(packages); int partitionCount = 100; int count = packages.Count; int chunkSize = count / (partitionCount - 1); IDictionary>[] partitions = new Dictionary>[partitionCount]; IDictionary> batch = new Dictionary>(); int batchNumber = 0; foreach (KeyValuePair> package in packages) { batch.Add(package); if (batch.Count == chunkSize) { partitions[batchNumber++] = new Dictionary>(batch); batch.Clear(); } } if (batch.Count > 0) { partitions[batchNumber++] = new Dictionary>(batch); } //CloudStorageAccount account = new CloudStorageAccount(new StorageCredentials("partitions", ""), false); IList createTasks = new List(); int partitionNumber = 0; foreach (IDictionary> partition in partitions) { string name = string.Format("partition{0}", partitionNumber++); Storage storage = new FileStorage("http://localhost:8000/partition/" + name, @"c:\data\site\partition\" + name); //Storage storage = new AzureStorage(account, name); createTasks.Add(IndexingHelpers.CreateNewCatalog(storage, partition)); } Task.WaitAll(createTasks.ToArray()); } public static void Test1() { string template = "http://partitions.blob.core.windows.net/partition{0}/index.json"; IList>>> tasks = new List>>>(); for (int i = 0; i < 100; i++) { Uri indexUri = new Uri(string.Format(template, i)); tasks.Add(IndexingHelpers.GetPackages(indexUri, true)); } Task.WaitAll(tasks.ToArray()); int packageCount = 0; int packageVersionCount = 0; foreach (Task>> task in tasks) { IDictionary> result = task.Result; packageCount += result.Count; foreach (KeyValuePair> package in result) { packageVersionCount += package.Value.Count; } } Console.WriteLine(packageCount); Console.WriteLine(packageVersionCount); } } }