You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
NuGetGallery/tests/CatalogTests/PartitioningTests.cs

136 lines
4.5 KiB
C#

using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Auth;
using Newtonsoft.Json.Linq;
using NuGet.Services.Metadata.Catalog.Helpers;
using NuGet.Services.Metadata.Catalog.Persistence;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
namespace CatalogTests
{
class PartitioningTests
{
static void PrintMetrics(IDictionary<string, IList<JObject>> packages)
{
Console.WriteLine();
Console.WriteLine(packages.Count);
int count = 0;
foreach (var versions in packages.Values)
{
count += versions.Count;
}
Console.WriteLine(count);
}
static async Task<HashSet<string>> GetPopularPackages()
{
Uri reportUri = new Uri("https://nugetgallery.blob.core.windows.net/stats/popularity/recentpopularity.json");
HttpClient client = new HttpClient();
Console.WriteLine(reportUri);
HttpResponseMessage reportResponse = await client.GetAsync(reportUri);
string reportJson = await reportResponse.Content.ReadAsStringAsync();
JToken report = JToken.Parse(reportJson);
HashSet<string> packages = new HashSet<string>();
foreach (JObject item in report)
{
string packageId = item["PackageId"].ToString().ToLowerInvariant();
packages.Add(packageId);
}
return packages;
}
public static void Test0()
{
Uri indexUri = new Uri("http://nugetprod0.blob.core.windows.net/ng-catalogs/0/index.json");
IDictionary<string, IList<JObject>> packages = IndexingHelpers.GetPackages(indexUri, true).Result;
PrintMetrics(packages);
int partitionCount = 100;
int count = packages.Count;
int chunkSize = count / (partitionCount - 1);
IDictionary<string, IList<JObject>>[] partitions = new Dictionary<string, IList<JObject>>[partitionCount];
IDictionary<string, IList<JObject>> batch = new Dictionary<string, IList<JObject>>();
int batchNumber = 0;
foreach (KeyValuePair<string, IList<JObject>> package in packages)
{
batch.Add(package);
if (batch.Count == chunkSize)
{
partitions[batchNumber++] = new Dictionary<string, IList<JObject>>(batch);
batch.Clear();
}
}
if (batch.Count > 0)
{
partitions[batchNumber++] = new Dictionary<string, IList<JObject>>(batch);
}
//CloudStorageAccount account = new CloudStorageAccount(new StorageCredentials("partitions", ""), false);
IList<Task> createTasks = new List<Task>();
int partitionNumber = 0;
foreach (IDictionary<string, IList<JObject>> partition in partitions)
{
string name = string.Format("partition{0}", partitionNumber++);
Storage storage = new FileStorage("http://localhost:8000/partition/" + name, @"c:\data\site\partition\" + name);
//Storage storage = new AzureStorage(account, name);
createTasks.Add(IndexingHelpers.CreateNewCatalog(storage, partition));
}
Task.WaitAll(createTasks.ToArray());
}
public static void Test1()
{
string template = "http://partitions.blob.core.windows.net/partition{0}/index.json";
IList<Task<IDictionary<string, IList<JObject>>>> tasks = new List<Task<IDictionary<string, IList<JObject>>>>();
for (int i = 0; i < 100; i++)
{
Uri indexUri = new Uri(string.Format(template, i));
tasks.Add(IndexingHelpers.GetPackages(indexUri, true));
}
Task.WaitAll(tasks.ToArray());
int packageCount = 0;
int packageVersionCount = 0;
foreach (Task<IDictionary<string, IList<JObject>>> task in tasks)
{
IDictionary<string, IList<JObject>> result = task.Result;
packageCount += result.Count;
foreach (KeyValuePair<string, IList<JObject>> package in result)
{
packageVersionCount += package.Value.Count;
}
}
Console.WriteLine(packageCount);
Console.WriteLine(packageVersionCount);
}
}
}