Merge Dev into Master (#109)

* ignore excess nodes in NuSpec in XML transform (#103)
* Upgrade to latest version of bin-skim utility (#105)
* Memory leak Fix (#106)
* Wrap RAMDirectory to correct FileModified return. Standardize file modified comparisons
* adding HSTS header to search responses (#108)

.nuget/packages.config

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="Microsoft.ApplicationInsights.JavaScript" version="0.11.0-build09387" />
-  <package id="Microsoft.CodeAnalysis.BinSkim" version="1.3.4-beta" />
+  <package id="Microsoft.CodeAnalysis.BinSkim" version="1.3.6" />
   <package id="xunit.runner.console" version="2.1.0" />
 </packages>

src/NuGet.Indexing/AzureDirectorySynchronizer.cs

@@ -104,7 +104,7 @@ private static void UnidirectionalSync(AzureDirectory sourceDirectory, Directory
 
             // we'll remove old files from both AzureDirectory's cache directory, as well as our destination directory
             // (only when older than 45 minutes - old files may still have active searches on them so we need a margin)
-            var referenceTimestamp = LuceneTimestampFromDateTime(DateTimeOffset.UtcNow.AddMinutes(-45));
+            var referenceTimestamp = LuceneTimestampFromDateTime(DateTime.UtcNow.AddMinutes(-45));
 
             // remove old files from AzureDirectory cache directory
             RemoveOldFiles(sourceDirectory.CacheDirectory, sourceFiles, referenceTimestamp);
@@ -126,11 +126,10 @@ private static void RemoveOldFiles(Directory directory, string[] skipFiles, long
             }
         }
 
-        private static long LuceneTimestampFromDateTime(DateTimeOffset date)
+        private static long LuceneTimestampFromDateTime(DateTime date)
         {
-            var epoch = new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero);
-
-            return (date.UtcTicks - epoch.UtcTicks) / TimeSpan.TicksPerSecond * 1000;
+            // Use ToFileTimeUtc here to stay consistent with the returns from AzureDirectory.
+            return date.ToFileTimeUtc();
         }
     }
 }

src/NuGet.Indexing/NuGet.Indexing.csproj

@@ -234,6 +234,7 @@
     <Compile Include="Querys\DownloadsBoostedQuery.cs" />
     <Compile Include="Querys\DownloadsScoreProvider.cs" />
     <Compile Include="Querys\QueryBoostingContext.cs" />
+    <Compile Include="RAMDirectoryWrapper.cs" />
     <Compile Include="Ranking.cs" />
     <Compile Include="RankingBySegment.cs" />
     <Compile Include="RankingResult.cs" />

src/NuGet.Indexing/NuGetSearcherManager.cs

@@ -121,7 +121,7 @@ public class NuGetSearcherManager : SearcherManager<NuGetIndexSearcher>
                 if (directory == null)
                 {
                     var sourceDirectory = new AzureDirectory(storageAccount, indexContainer);
-                    directory = new RAMDirectory(sourceDirectory); // initial copy from storage to RAM
+                    directory = new RAMDirectoryWrapper(sourceDirectory); // initial copy from storage to RAM
 
                     azureDirectorySynchronizer = new AzureDirectorySynchronizer(sourceDirectory, directory);
                 }

src/NuGet.Indexing/RAMDirectoryWrapper.cs

@@ -0,0 +1,36 @@
+using Lucene.Net.Store;
+using System;
+
+namespace NuGet.Indexing
+{
+    /// <summary>
+    /// This class wraps the RAMDirectory so we can correct the return of FileModified function
+    /// </summary>
+    public class RAMDirectoryWrapper: RAMDirectory
+    {
+        public RAMDirectoryWrapper() : base() { }
+
+        public RAMDirectoryWrapper(Directory seedDirectory) : base(seedDirectory) { }
+
+        /// <summary>
+        /// Returns the time (as a long) the named file was last modified in a Windows FileTime in UTC
+        /// </summary>
+        /// <param name="name">File Name</param>
+        /// <returns>A long that represents a UTC Windows FileTime</returns>
+        /// <remarks>The implementation here is to keep in line with the implementation in AzureDirectory for use in the <see cref="AzureDirectorySynchronizer"/>.
+        /// See https://github.com/azure-contrib/AzureDirectory/blob/master/AzureDirectory/AzureDirectory.cs#L147 for AzureDirectory implementation</remarks>
+        public override long FileModified(string name)
+        {
+            // The RAMDirectory implementation of FileModified creates a dateTime, converts to localTime, and then uses ticks to get milliseconds.
+            // Undo this conversion here so we can get the file modified time in ticks back (accurate to the millisecond, we lose some precision here)
+            // Then operate on this to get back a "standard" utc windows filetime.
+            // See https://lucenenet.apache.org/docs/3.0.3/d7/df5/_r_a_m_directory_8cs_source.html line 114 for more details.
+
+            var originalTicks = base.FileModified(name) * TimeSpan.TicksPerMillisecond;
+
+            var newTime = new DateTime(originalTicks, DateTimeKind.Local);
+
+            return newTime.ToUniversalTime().ToFileTimeUtc();
+        }
+    }
+}

src/NuGet.Services.BasicSearch/Startup.cs

@@ -71,6 +71,13 @@ public void Configuration(IAppBuilder app, IConfiguration configuration, Directo
             // Add Application Insights
             app.Use(typeof(RequestTrackingMiddleware));
 
+            // Enable HSTS
+            app.Use(async (context, next) =>
+            {
+                context.Response.Headers.Add("Strict-Transport-Security", new string[] { "max-age=31536000; includeSubDomains" });
+                await next.Invoke();
+            });
+
             // Enable CORS
             var corsPolicy = new CorsPolicy
             {

tests/NuGet.Services.BasicSearchTests/TestSupport/LuceneDirectoryInitializer.cs

@@ -59,7 +59,7 @@ private Lucene.Net.Store.Directory CreateLuceneIndex(IEnumerable<PackageVersion>
             }
             else
             {
-                directory = new RAMDirectory();
+                directory = new RAMDirectoryWrapper();
             }
             
             using (var indexWriter = DocumentCreator.CreateIndexWriter(directory, true))