versaplexd/vxdiskschema.cs

   1 /*
   2  * Versaplex:
   3  *   Copyright (C)2007-2008 Versabanq Innovations Inc. and contributors.
   4  *       See the included file named LICENSE for license information.
   5  */
   6 using System;
   7 using System.Collections;
   8 using System.Collections.Generic;
   9 using System.IO;
  10 using System.Text;
  11 using System.Security.Cryptography;
  12 using Wv;
  13 using Wv.Extensions;
  14
  15 // An ISchemaBackend that uses a directory on disk as a backing store.
  16 [WvMoniker]
  17 internal class VxDiskSchema : ISchemaBackend
  18 {
  19     static WvLog log = new WvLog("VxDiskSchema", WvLog.L.Debug2);
  20
  21     private string exportdir;
  22
  23     public static void wvmoniker_register()
  24     {
  25         WvMoniker<ISchemaBackend>.register("dir",
  26                   (string m, object o) => new VxDiskSchema(m));
  27     }
  28
  29     public VxDiskSchema(string _exportdir)
  30     {
  31         exportdir = _exportdir;
  32     }
  33
  34     public void Dispose()
  35     {
  36     }
  37
  38     //
  39     // The ISchemaBackend interface
  40     //
  41
  42     // Export the current schema to the backing directory, in a format that can
  43     // be read back later.
  44     public VxSchemaErrors Put(VxSchema schema, VxSchemaChecksums sums,
  45         VxPutOpts opts)
  46     {
  47         bool isbackup = (opts & VxPutOpts.IsBackup) != 0;
  48
  49         DirectoryInfo dir = new DirectoryInfo(exportdir);
  50         dir.Create();
  51
  52         foreach (var p in schema)
  53         {
  54             if (!sums.ContainsKey(p.Key))
  55                 throw new ArgumentException("Missing checksum for " + p.Key);
  56
  57             VxSchemaElement elem = p.Value;
  58             if (elem.text == null || elem.text == "")
  59                 DropSchema(new string[] {elem.key});
  60             else
  61                 ExportToDisk(p.Value, sums[p.Key], isbackup);
  62         }
  63
  64         // Writing schemas to disk doesn't give us any per-element errors.
  65         return new VxSchemaErrors();
  66     }
  67
  68     public VxSchema Get(IEnumerable<string> keys)
  69     {
  70         VxSchema fullschema = new VxSchema();
  71         VxSchema schema = new VxSchema();
  72
  73         ReadExportedDir(fullschema, null);
  74
  75         if (keys == null)
  76             return fullschema;
  77
  78         // This is a bit slow and stupid - we could just read only the
  79         // required keys from disk.  But the key-limiting is mainly for the
  80         // much slower dbus and database backends, so it's probably not worth
  81         // fixing.
  82         foreach (string key in keys)
  83             schema.Add(key, fullschema[key]);
  84
  85         if (schema.Count == 0)
  86             schema = fullschema;
  87
  88         return schema;
  89     }
  90
  91     public VxSchemaChecksums GetChecksums()
  92     {
  93         VxSchemaChecksums sums = new VxSchemaChecksums();
  94         ReadExportedDir(null, sums);
  95         return sums;
  96     }
  97
  98     public VxSchemaErrors DropSchema(IEnumerable<string> keys)
  99     {
 100         VxSchemaErrors errs = new VxSchemaErrors();
 101
 102         foreach (string key in keys)
 103         {
 104             string fullpath = wv.PathCombine(exportdir, key);
 105             log.print("Removing {0}\n", fullpath);
 106             if (File.Exists(fullpath))
 107                 File.Delete(fullpath);
 108             if (key.StartsWith("Index/"))
 109             {
 110                 string type, name;
 111                 VxSchema.ParseKey(key, out type, out name);
 112                 if (type != "Index")
 113                     continue;
 114
 115                 // If it was the last index for a table, remove the empty dir.
 116                 string[] split = wv.PathSplit(name);
 117                 if (split.Length > 0)
 118                 {
 119                     string table = split[0];
 120                     string tabpath = wv.PathCombine(exportdir, type, table);
 121                     // Directory.Delete won't delete non-empty dirs, but we
 122                     // still check both for safety and to write a sensible
 123                     // message.
 124                     if (Directory.GetFileSystemEntries(tabpath).Length == 0)
 125                     {
 126                         log.print("Removing empty directory {0}\n", tabpath);
 127                         Directory.Delete(tabpath);
 128                     }
 129                 }
 130             }
 131         }
 132
 133         return errs;
 134     }
 135
 136     // Note: we ignore the "where" clause and just return everything.
 137     public string GetSchemaData(string tablename, int seqnum, string where,
 138                 Dictionary<string,string> replaces, List<string> skipfields)
 139     {
 140         string datadir = Path.Combine(exportdir, "DATA");
 141         string filename = wv.fmt("{0}-{1}.sql", seqnum, tablename);
 142         string fullpath = Path.Combine(datadir, filename);
 143
 144         return File.ReadAllText(fullpath);
 145     }
 146
 147     public void PutSchemaData(string tablename, string text, int seqnum)
 148     {
 149         string datadir = Path.Combine(exportdir, "DATA");
 150         string filename = wv.fmt("{0}-{1}.sql", seqnum, tablename);
 151         string fullpath = Path.Combine(datadir, filename);
 152
 153         Directory.CreateDirectory(datadir);
 154         File.WriteAllBytes(fullpath, text.ToUTF8());
 155     }
 156
 157     //
 158     // Non-ISchemaBackend methods
 159     //
 160
 161     // Retrieves both the schema and its checksums from exportdir, and puts
 162     // them into the parameters.
 163     private void ReadExportedDir(VxSchema schema, VxSchemaChecksums sums)
 164     {
 165         DirectoryInfo exportdirinfo = new DirectoryInfo(exportdir);
 166         if (exportdirinfo.Exists)
 167         {
 168             // Read all files that match */* and Index/*/*.
 169             foreach (DirectoryInfo dir1 in exportdirinfo.GetDirectories())
 170             {
 171                 if (dir1.Name == "DATA")
 172                     continue;
 173
 174                 string type = dir1.Name;
 175
 176                 foreach (DirectoryInfo dir2 in dir1.GetDirectories())
 177                 {
 178                     if (dir2.Name == "DATA" || dir1.Name != "Index")
 179                         continue;
 180
 181                     // This is the Index/*/* part
 182                     foreach (FileInfo file in dir2.GetFiles())
 183                     {
 184                         if (!IsFileNameUseful(file.Name))
 185                             continue;
 186
 187                         string name = wv.PathCombine(dir2.Name, file.Name);
 188                         AddFromFile(file.FullName, type, name, schema, sums);
 189                     }
 190                 }
 191
 192                 // This is the */* part
 193                 foreach (FileInfo file in dir1.GetFiles())
 194                 {
 195                     if (!IsFileNameUseful(file.Name))
 196                         continue;
 197
 198                     AddFromFile(file.FullName, type, file.Name, schema, sums);
 199                 }
 200             }
 201         }
 202     }
 203
 204     // Static methods
 205
 206     // We want to ignore hidden files, and backup files left by editors.
 207     private static bool IsFileNameUseful(string filename)
 208     {
 209         return !filename.StartsWith(".") && !filename.EndsWith("~");
 210     }
 211
 212     // Adds the contents of extradir to the provided schema and sums.
 213     // Throws an ArgumentException if the directory contains an entry that
 214     // already exists in schema or sums.
 215     public static void AddFromDir(string extradir, VxSchema schema,
 216         VxSchemaChecksums sums)
 217     {
 218         VxDiskSchema disk = new VxDiskSchema(extradir);
 219
 220         disk.ReadExportedDir(schema, sums);
 221     }
 222
 223     // Reads a file from an on-disk exported schema, and sets the schema
 224     // element parameter's text field, if the schema element isn't null.
 225     // Returns a new VxSchemaChecksum object containing the checksum.
 226     // Returns true if the file passes its MD5 validation.
 227     // If it returns false, elem and sum may be set to null.
 228     private static bool ReadSchemaFile(string filename, string type,
 229         string name, out VxSchemaElement elem, out VxSchemaChecksum sum)
 230     {
 231         elem = null;
 232         sum = null;
 233
 234         FileInfo fileinfo = new FileInfo(filename);
 235
 236         // Read the entire file into memory.  C#'s file IO sucks.
 237         byte[] bytes = File.ReadAllBytes(filename);
 238
 239         // Find the header line
 240         int ii;
 241         for (ii = 0; ii < bytes.Length; ii++)
 242             if (bytes[ii] == '\n')
 243                 break;
 244
 245         if (ii == bytes.Length)
 246             return false;
 247
 248         // Read the header line
 249         Encoding utf8 = Encoding.UTF8;
 250         string header = utf8.GetString(bytes, 0, ii).Replace("\r", "");
 251
 252         // Skip the newline
 253         if (bytes[ii] == '\n')
 254             ii++;
 255
 256         // Read the body
 257         string body = utf8.GetString(bytes, ii, bytes.Length - ii);
 258         elem = VxSchemaElement.create(type, name, body, false);
 259
 260         // Parse the header line
 261         char[] space = {' '};
 262         string[] headers = header.Split(space, 3);
 263
 264         string prefix = headers[0];
 265         string header_md5 = headers.Length >= 2 ? headers[1] : "";
 266         string dbsum = headers.Length >= 3 ? headers[2] : "";
 267
 268         if (prefix != "!!SCHEMAMATIC")
 269             return false;
 270
 271         // Compute the hash of the rest of the file
 272         byte[] md5 = MD5.Create().ComputeHash(bytes, ii,
 273             (int)fileinfo.Length - ii);
 274         string content_md5 = md5.ToHex().ToLower();
 275
 276         IEnumerable<ulong> sumlist;
 277
 278         // If the MD5 sums don't match, we want to make it obvious that the
 279         // database and local file aren't in sync, so we don't load any actual
 280         // checksums.
 281         if (String.Compare(header_md5, content_md5, true) == 0)
 282         {
 283             string errctx = wv.fmt("Error while reading file {0}: ", filename);
 284             sumlist = VxSchemaChecksum.ParseSumString(dbsum, errctx);
 285         }
 286         else
 287         {
 288             log.print(WvLog.L.Info, "Checksum mismatch for {0}\n", filename);
 289             sumlist = new List<ulong>();
 290         }
 291
 292         sum = new VxSchemaChecksum(elem.key, sumlist);
 293         return true;
 294     }
 295
 296     // Helper method to load a given on-disk element's schema and checksums
 297     // into the container objects.
 298     // Throws an ArgumentException if the schema or sums already contains the
 299     // given key.
 300     private static void AddFromFile(string path, string type, string name,
 301         VxSchema schema, VxSchemaChecksums sums)
 302     {
 303         string key = wv.fmt("{0}/{1}", type, name);
 304
 305         // schema/sums.Add would throw an exception in this situation anyway,
 306         // but it's nice to provide a more helpful error message.
 307         if (schema != null && schema.ContainsKey(key))
 308             throw new ArgumentException("Conflicting schema key: " + key);
 309         if (sums != null && sums.ContainsKey(key))
 310             throw new ArgumentException("Conflicting sums key: " + key);
 311
 312         VxSchemaChecksum sum;
 313         VxSchemaElement elem;
 314         ReadSchemaFile(path, type, name, out elem, out sum);
 315
 316         if (schema != null && elem != null)
 317             schema.Add(key, elem);
 318         if (sums != null && sum != null)
 319             sums.Add(key, sum);
 320     }
 321
 322     private void ExportToDisk(VxSchemaElement elem, VxSchemaChecksum sum,
 323         bool isbackup)
 324     {
 325         // Make some kind of attempt to run on Windows.
 326         string filename = wv.PathJoin(exportdir, elem.type, elem.name);
 327
 328         // Make directories
 329         Directory.CreateDirectory(Path.GetDirectoryName(filename));
 330
 331         string suffix = "";
 332         if (isbackup)
 333         {
 334             int i = 1;
 335             while(File.Exists(filename + "-" + i))
 336                 i++;
 337             suffix = "-" + i;
 338         }
 339
 340         filename += suffix;
 341
 342         log.print("Writing {0}\n", filename);
 343         File.WriteAllBytes(filename, elem.ToStringWithHeader(sum).ToUTF8());
 344     }
 345
 346 }
 347