tools/nuget-hash-extractor/nuget-hash-extractor.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.Collections.Specialized;
   4 using System.Xml.Linq;
   5 using System.Linq;
   6 using System.IO;
   7 using System.IO.Compression;
   8 using System.Reflection;
   9 using System.Text;
  10
  11 class Driver {
  12
  13         internal static Dictionary<string, string> BadAssembliesToEnumTable = new Dictionary<string, string> {
  14                 {"System.Runtime.InteropServices.RuntimeInformation.dll", "SYS_RT_INTEROP_RUNTIME_INFO"},
  15                 {"System.Globalization.Extensions.dll", "SYS_GLOBALIZATION_EXT"},
  16                 {"System.IO.Compression.dll", "SYS_IO_COMPRESSION"},
  17                 {"System.Net.Http.dll", "SYS_NET_HTTP"},
  18                 {"System.Text.Encoding.CodePages.dll", "SYS_TEXT_ENC_CODEPAGES"},
  19                 {"System.Reflection.DispatchProxy.dll", "SYS_REF_DISP_PROXY"},
  20                 {"System.Threading.Overlapped.dll", "SYS_THREADING_OVERLAPPED"}
  21         };
  22
  23         // IGNORED_ASSEMBLY
  24         internal static Dictionary<string, NuGetData> ignoredAsmTable = new Dictionary<string, NuGetData> ();
  25
  26         // IGNORED_ASM_VER
  27         internal static SortedDictionary<string, NuGetData> ignoredAsmVerTable = new SortedDictionary<string, NuGetData> ();
  28
  29         static ZipArchiveEntry FindSpecFile (ZipArchive zip) {
  30                 foreach (var entry in zip.Entries) {
  31                         if (entry.Name.EndsWith (".nuspec"))
  32                                 return entry;
  33                 }
  34                 throw new Exception ("Could not find nuspec file");
  35         }
  36
  37         static void DumpNuget (string nupkg) {
  38                 var zip = new ZipArchive(new FileStream (nupkg, FileMode.Open));
  39
  40                 var nuspec = FindSpecFile (zip);
  41                 var l = XElement.Load (new StreamReader (nuspec.Open ()));
  42                 var version = (from el in l.Descendants() where el.Name.LocalName == "version" select el.Value).FirstOrDefault ();
  43
  44                 var prefixesToCheckInOrder = new string[] {
  45                         "lib/net4",
  46                         "lib/netstandard1",
  47                         "msbuildExtensions/Microsoft/Microsoft.NET.Build.Extensions/net4"
  48                 };
  49
  50                 //
  51                 // Logic copied from https://github.com/NuGet/NuGet.Client/blob/4cccb13833ad29d6a0bcff055460d964f1b49cfe/src/NuGet.Core/NuGet.Frameworks/DefaultFrameworkMappings.cs#L385
  52                 //
  53                 IEnumerable<ZipArchiveEntry> entries = null;
  54                 foreach (var prefix in prefixesToCheckInOrder) {
  55                         entries = zip.Entries.Where (e => e.FullName.StartsWith (prefix) && !e.FullName.Contains ("/ref/") && e.Name.EndsWith (".dll") && BadAssembliesToEnumTable.ContainsKey (e.Name));
  56                         if (entries.Any ())
  57                                 break;
  58                 }
  59
  60                 if (!entries.Any ()) {
  61                         Console.Error.WriteLine ($"** Warning: No relevant assemblies found for nukpkg: {nupkg}");
  62                         return;
  63                 }
  64
  65                 // Only take assemblies from first prefix
  66                 foreach (var et in entries) {
  67                         LoadAndDump (et, version);
  68                 }
  69         }
  70
  71         static void DumpOutput ()
  72         {
  73                 if (dump_asm) {
  74                         var sb = new StringBuilder();
  75                         // sorted just by hashcode
  76                         foreach (var data in ignoredAsmTable.Values.OrderBy (data => data.HashCode)) {
  77                                 sb.AppendLine ($"IGNORED_ASSEMBLY (0x{data.HashCode}, {data.EnumName}, \"{data.ModuleVersionId}\", \"{data.NuGetVersionAndFramework}\"),");
  78                         }
  79
  80                         if (sb.Length > 2)
  81                                 sb.Length = sb.Length - 2;
  82                         Console.WriteLine (sb.ToString ());
  83                 }
  84
  85                 //IGNORED_ASM_VER (SYS_IO_COMPRESSION, 4, 1, 2, 0),
  86                 if (dump_ver) {
  87                         // the default key (IgnoredAsmVerKey) is the order which we need for this
  88                         var sb = new StringBuilder();
  89                         foreach (var key in ignoredAsmVerTable.Keys) {
  90                                 var data = ignoredAsmVerTable [key];
  91                                 var ver = data.AssemblyVersion;
  92                                 sb.AppendLine ($"IGNORED_ASM_VER ({data.EnumName}, {ver.Major}, {ver.Minor}, {ver.Build}, {ver.Revision}),");
  93                         }
  94
  95                         if (sb.Length > 2)
  96                                 sb.Length = sb.Length - 2;
  97                         Console.WriteLine (sb.ToString ());
  98                 }
  99
 100                 if (dump_guids_for_msbuild) {
 101                         // This needs to be kept in sync with FilterDeniedAssemblies msbuild task in msbuild
 102                         // Sory by assembly name and version
 103                         var query = ignoredAsmTable.Values
 104                                                 .GroupBy (data => data.AssemblyName)
 105                                                 .Select (group => new { AssemblyName = group.Key, NuGets = group.OrderBy (data => data.AssemblyVersion) });
 106
 107                         foreach (var g in query) {
 108                                 foreach (var data in g.NuGets) {
 109                                         var ver = data.AssemblyVersion;
 110                                         Console.WriteLine ($"denied:{data.AssemblyName},{data.ModuleVersionId},{ver.Major},{ver.Minor},{ver.Build},{ver.Revision}");
 111                                 }
 112                         }
 113                 }
 114         }
 115
 116         static bool dump_asm, dump_ver, dump_guids_for_msbuild;
 117         static void Main (string[] args) {
 118
 119                 if (args.Length > 1) {
 120                         dump_asm = args [1].Equals ("asm");
 121                         dump_ver = args [1].Equals ("ver");
 122                         dump_guids_for_msbuild = args [1].Equals ("guids_for_msbuild");
 123
 124                         if (args [1].Equals("all"))
 125                                 dump_asm = dump_ver = dump_guids_for_msbuild = true;
 126                 } else {
 127                         dump_asm = true;
 128                 }
 129                 foreach (var f in Directory.GetFiles (args [0], "*.nupkg")) {
 130                         DumpNuget (f);
 131                 }
 132
 133                 DumpOutput ();
 134         }
 135
 136         static byte[] StreamToArray (Stream s) {
 137                 using(var ms = new MemoryStream ()) {
 138                         s.CopyTo (ms);
 139                         return ms.ToArray ();
 140                 }
 141         }
 142
 143         static int domain_id = 1;
 144         static void LoadAndDump (ZipArchiveEntry entry, string version) {
 145                 var data = StreamToArray (entry.Open ());
 146                 AppDomain ad = AppDomain.CreateDomain ("parse_" + ++domain_id);
 147                 DoParse p = (DoParse)ad.CreateInstanceAndUnwrap (typeof (DoParse).Assembly.FullName, typeof (DoParse).FullName);
 148                 var nugetData = p.ParseAssembly (data, version, entry.Name, entry.FullName);
 149
 150                 if (!ignoredAsmTable.ContainsKey (nugetData.IgnoredAsmKey))
 151                         ignoredAsmTable [nugetData.IgnoredAsmKey] = nugetData;
 152
 153                 if (!ignoredAsmVerTable.ContainsKey (nugetData.IgnoredAsmVerKey))
 154                         ignoredAsmVerTable [nugetData.IgnoredAsmVerKey] = nugetData;
 155
 156                 AppDomain.Unload (ad);
 157         }
 158 }
 159
 160 class DoParse : MarshalByRefObject {
 161
 162         static int Hash (string str) {
 163                 int h = 5381;
 164         for (int i = 0;  i < str.Length; ++i)
 165             h = ((h << 5) + h) ^ str[i];
 166                 return h;
 167         }
 168
 169         static string FileToMoniker (string path) =>
 170                                         path.Split (Path.DirectorySeparatorChar)
 171                                                 .Where (p => p.StartsWith ("net"))
 172                                                 .FirstOrDefault ()
 173                                         ?? "unknown";
 174
 175         static string FileToEnum (string name) =>
 176                                         Driver.BadAssembliesToEnumTable.ContainsKey (name)
 177                                                 ? Driver.BadAssembliesToEnumTable [name]
 178                                                 : throw new Exception ($"No idea what to do with {name}");
 179
 180         public NuGetData ParseAssembly (byte[] data, string version, string name, string fullname) {
 181                 var a = Assembly.ReflectionOnlyLoad (data);
 182                 var m = a.GetModules ()[0];
 183                 var id = m.ModuleVersionId.ToString ().ToUpper ();
 184
 185                 return new NuGetData {
 186                         AssemblyName = name,
 187                         EnumName = FileToEnum (name),
 188                         NuGetVersionAndFramework = version + " " + FileToMoniker (fullname),
 189                         AssemblyVersion = a.GetName ().Version,
 190                         HashCode = Hash (id).ToString ("X"),
 191                         ModuleVersionId = id
 192                 };
 193         }
 194 }
 195
 196 [Serializable]
 197 class NuGetData
 198 {
 199         public string AssemblyName;
 200         public string EnumName;
 201         public string NuGetVersionAndFramework;
 202         public Version AssemblyVersion;
 203         public string HashCode;
 204         public string ModuleVersionId;
 205
 206         public string IgnoredAsmKey => $"{HashCode},{EnumName},{ModuleVersionId}";
 207         public string IgnoredAsmVerKey => $"{EnumName},{AssemblyVersion.ToString()}";
 208 }