tools/nuget-hash-extractor/nuget-hash-extractor.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.Collections.Specialized;
   4 using System.Xml.Linq;
   5 using System.Linq;
   6 using System.IO;
   7 using System.IO.Compression;
   8 using System.Reflection;
   9 using System.Text;
  10
  11 class Driver {
  12
  13         internal static Dictionary<string, string> BadAssembliesToEnumTable = new Dictionary<string, string> {
  14                 {"System.Runtime.InteropServices.RuntimeInformation.dll", "SYS_RT_INTEROP_RUNTIME_INFO"},
  15                 {"System.Globalization.Extensions.dll", "SYS_GLOBALIZATION_EXT"},
  16                 {"System.IO.Compression.dll", "SYS_IO_COMPRESSION"},
  17                 {"System.Net.Http.dll", "SYS_NET_HTTP"},
  18                 {"System.Text.Encoding.CodePages.dll", "SYS_TEXT_ENC_CODEPAGES"},
  19                 {"System.Reflection.DispatchProxy.dll", "SYS_REF_DISP_PROXY"},
  20                 {"System.Threading.Overlapped.dll", "SYS_THREADING_OVERLAPPED"}
  21         };
  22
  23         // IGNORED_ASSEMBLY
  24         internal static Dictionary<string, NuGetData> ignoredAsmTable = new Dictionary<string, NuGetData> ();
  25
  26         // IGNORED_ASM_VER
  27         internal static SortedDictionary<string, NuGetData> ignoredAsmVerTable = new SortedDictionary<string, NuGetData> ();
  28
  29         static ZipArchiveEntry FindSpecFile (ZipArchive zip) {
  30                 foreach (var entry in zip.Entries) {
  31                         if (entry.Name.EndsWith (".nuspec"))
  32                                 return entry;
  33                 }
  34                 throw new Exception ("Could not find nuspec file");
  35         }
  36
  37         static void DumpNuget (string nupkg) {
  38                 var zip = new ZipArchive(new FileStream (nupkg, FileMode.Open));
  39
  40                 var nuspec = FindSpecFile (zip);
  41                 var l = XElement.Load (new StreamReader (nuspec.Open ()));
  42                 var version = (from el in l.Descendants() where el.Name.LocalName == "version" select el.Value).FirstOrDefault ();
  43
  44                 var prefixesToCheckInOrder = new string[] {
  45                         "lib/net4",
  46                         "lib/netstandard1",
  47                         "msbuildExtensions/Microsoft/Microsoft.NET.Build.Extensions/net4"
  48                 };
  49
  50                 //
  51                 // Logic copied from https://github.com/NuGet/NuGet.Client/blob/4cccb13833ad29d6a0bcff055460d964f1b49cfe/src/NuGet.Core/NuGet.Frameworks/DefaultFrameworkMappings.cs#L385
  52                 //
  53                 IEnumerable<ZipArchiveEntry> entries = null;
  54                 foreach (var prefix in prefixesToCheckInOrder) {
  55                         entries = zip.Entries.Where (e => e.FullName.StartsWith (prefix) && !e.FullName.Contains ("/ref/") && e.Name.EndsWith (".dll") && BadAssembliesToEnumTable.ContainsKey (e.Name));
  56                         if (entries.Any ())
  57                                 break;
  58                 }
  59
  60                 if (!entries.Any ()) {
  61                         Console.Error.WriteLine ($"** Warning: No relevant assemblies found for nukpkg: {nupkg}");
  62                         return;
  63                 }
  64
  65                 // Only take assemblies from first prefix
  66                 foreach (var et in entries) {
  67                         LoadAndDump (et, version);
  68                 }
  69         }
  70
  71         static void DumpOutput ()
  72         {
  73                 if (dump_asm) {
  74                         var sb = new StringBuilder();
  75                         // sorted just by hashcode
  76                         foreach (var data in ignoredAsmTable.Values.OrderBy (data => data.HashCode)) {
  77                                 sb.AppendLine ($"IGNORED_ASSEMBLY (0x{data.HashCode}, {data.EnumName}, \"{data.ModuleVersionId}\", \"{data.NuGetVersionAndFramework}\"),");
  78                         }
  79
  80                         if (sb.Length > 2)
  81                                 sb.Length = sb.Length - 2;
  82                         Console.WriteLine (sb.ToString ());
  83                 }
  84
  85                 //IGNORED_ASM_VER (SYS_IO_COMPRESSION, 4, 1, 2, 0),
  86                 if (dump_ver) {
  87                         // the default key (IgnoredAsmVerKey) is the order which we need for this
  88                         var sb = new StringBuilder();
  89                         foreach (var key in ignoredAsmVerTable.Keys) {
  90                                 var data = ignoredAsmVerTable [key];
  91                                 var ver = data.AssemblyVersion;
  92                                 sb.AppendLine ($"IGNORED_ASM_VER ({data.EnumName}, {ver.Major}, {ver.Minor}, {ver.Build}, {ver.Revision}),");
  93                         }
  94
  95                         if (sb.Length > 2)
  96                                 sb.Length = sb.Length - 2;
  97                         Console.WriteLine (sb.ToString ());
  98                 }
  99
 100                 if (dump_guids_for_msbuild) {
 101                         // This needs to be kept in sync with FilterDeniedAssemblies msbuild task in msbuild
 102                         // Sory by assembly name and version
 103                         var query = ignoredAsmTable.Values
 104                                                 .GroupBy (data => data.AssemblyName)
 105                                                 .OrderBy (group => group.Key)
 106                                                 .Select (group => new { AssemblyName = group.Key, NuGets = group.OrderBy (data => data.AssemblyVersion).ThenBy (data => data.ModuleVersionId) });
 107
 108                         foreach (var g in query) {
 109                                 foreach (var data in g.NuGets) {
 110                                         var ver = data.AssemblyVersion;
 111                                         Console.WriteLine ($"denied:{data.AssemblyName},{data.ModuleVersionId},{ver.Major},{ver.Minor},{ver.Build},{ver.Revision}");
 112                                 }
 113                         }
 114                 }
 115         }
 116
 117         static bool dump_asm, dump_ver, dump_guids_for_msbuild;
 118         static void Main (string[] args) {
 119
 120                 if (args.Length > 1) {
 121                         dump_asm = args [1].Equals ("asm");
 122                         dump_ver = args [1].Equals ("ver");
 123                         dump_guids_for_msbuild = args [1].Equals ("guids_for_msbuild");
 124
 125                         if (args [1].Equals("all"))
 126                                 dump_asm = dump_ver = dump_guids_for_msbuild = true;
 127                 } else {
 128                         dump_asm = true;
 129                 }
 130                 foreach (var f in Directory.GetFiles (args [0], "*.nupkg").OrderBy (nupkg => Path.GetFileName (nupkg))) {
 131                         DumpNuget (f);
 132                 }
 133
 134                 DumpOutput ();
 135         }
 136
 137         static byte[] StreamToArray (Stream s) {
 138                 using(var ms = new MemoryStream ()) {
 139                         s.CopyTo (ms);
 140                         return ms.ToArray ();
 141                 }
 142         }
 143
 144         static int domain_id = 1;
 145         static void LoadAndDump (ZipArchiveEntry entry, string version) {
 146                 var data = StreamToArray (entry.Open ());
 147                 AppDomain ad = AppDomain.CreateDomain ("parse_" + ++domain_id);
 148                 DoParse p = (DoParse)ad.CreateInstanceAndUnwrap (typeof (DoParse).Assembly.FullName, typeof (DoParse).FullName);
 149                 var nugetData = p.ParseAssembly (data, version, entry.Name, entry.FullName);
 150
 151                 if (!ignoredAsmTable.ContainsKey (nugetData.IgnoredAsmKey))
 152                         ignoredAsmTable [nugetData.IgnoredAsmKey] = nugetData;
 153
 154                 if (!ignoredAsmVerTable.ContainsKey (nugetData.IgnoredAsmVerKey))
 155                         ignoredAsmVerTable [nugetData.IgnoredAsmVerKey] = nugetData;
 156
 157                 AppDomain.Unload (ad);
 158         }
 159 }
 160
 161 class DoParse : MarshalByRefObject {
 162
 163         static int Hash (string str) {
 164                 int h = 5381;
 165         for (int i = 0;  i < str.Length; ++i)
 166             h = ((h << 5) + h) ^ str[i];
 167                 return h;
 168         }
 169
 170         static string FileToMoniker (string path) =>
 171                                         path.Split (Path.DirectorySeparatorChar)
 172                                                 .Where (p => p.StartsWith ("net"))
 173                                                 .FirstOrDefault ()
 174                                         ?? "unknown";
 175
 176         static string FileToEnum (string name) =>
 177                                         Driver.BadAssembliesToEnumTable.ContainsKey (name)
 178                                                 ? Driver.BadAssembliesToEnumTable [name]
 179                                                 : throw new Exception ($"No idea what to do with {name}");
 180
 181         public NuGetData ParseAssembly (byte[] data, string version, string name, string fullname) {
 182                 var a = Assembly.ReflectionOnlyLoad (data);
 183                 var m = a.GetModules ()[0];
 184                 var id = m.ModuleVersionId.ToString ().ToUpper ();
 185
 186                 return new NuGetData {
 187                         AssemblyName = name,
 188                         EnumName = FileToEnum (name),
 189                         NuGetVersionAndFramework = version + " " + FileToMoniker (fullname),
 190                         AssemblyVersion = a.GetName ().Version,
 191                         HashCode = Hash (id).ToString ("X"),
 192                         ModuleVersionId = id
 193                 };
 194         }
 195 }
 196
 197 [Serializable]
 198 class NuGetData
 199 {
 200         public string AssemblyName;
 201         public string EnumName;
 202         public string NuGetVersionAndFramework;
 203         public Version AssemblyVersion;
 204         public string HashCode;
 205         public string ModuleVersionId;
 206
 207         public string IgnoredAsmKey => $"{HashCode},{EnumName},{ModuleVersionId}";
 208         public string IgnoredAsmVerKey => $"{EnumName},{AssemblyVersion.ToString()}";
 209 }