Allow schema files that are missing checksums on the !!SCHEMAMATIC line.
[versaplex.git] / versaplexd / vxschema.cs
blob887751358ab01eed7f3dae0be7b4f16948b5e51c
1 /*
2 * Versaplex:
3 * Copyright (C)2007-2008 Versabanq Innovations Inc. and contributors.
4 * See the included file named LICENSE for license information.
5 */
6 using System;
7 using System.IO;
8 using System.Text;
9 using System.Collections;
10 using System.Collections.Generic;
11 using System.Security.Cryptography;
12 using Wv;
13 using Wv.Extensions;
15 [Flags]
16 public enum VxCopyOpts : int
18 None = 0,
19 DryRun = 0x1,
20 ShowProgress = 0x2,
21 ShowDiff = 0x4,
22 Destructive = 0x8,
24 Verbose = ShowProgress | ShowDiff,
27 internal class VxSchemaElement : IComparable
29 string _type;
30 public string type {
31 get { return _type; }
34 string _name;
35 public string name {
36 get { return _name; }
39 string _text;
40 public virtual string text {
41 get { return _text; }
42 set { _text = value;}
45 bool _encrypted;
46 public bool encrypted {
47 get { return _encrypted; }
50 public string key {
51 get { return type + "/" + name; }
54 public static VxSchemaElement create(string type, string name,
55 string text, bool encrypted)
57 try {
58 if (type == "Table")
59 return new VxSchemaTable(name, text);
60 } catch (ArgumentException) {
61 // if the table data is invalid, just ignore it.
62 // We'll fall through and load a VxSchemaElement instead.
65 return new VxSchemaElement(type, name, text, encrypted);
68 protected VxSchemaElement(string newtype, string newname,
69 string newtext, bool newencrypted)
71 _type = newtype;
72 _name = newname;
73 _encrypted = newencrypted;
74 _text = newtext;
77 public static VxSchemaElement create(VxSchemaElement copy)
79 return create(copy.type, copy.name, copy.text, copy.encrypted);
82 public static VxSchemaElement create(IEnumerable<WvAutoCast> _elem)
84 var elem = _elem.GetEnumerator();
85 return create(elem.pop(), elem.pop(), elem.pop(), elem.pop() > 0);
88 public void Write(WvDbusWriter writer)
90 writer.Write(type);
91 writer.Write(name);
92 writer.Write(text);
93 byte encb = (byte)(encrypted ? 1 : 0);
94 writer.Write(encb);
97 public string GetKey()
99 return VxSchema.GetKey(type, name, encrypted);
102 // It's not guaranteed that the text field will be valid SQL. Give
103 // subclasses a chance to translate.
104 public virtual string ToSql()
106 return this.text;
109 // Returns the element's text, along with a header line containing the MD5
110 // sum of the text, and the provided database checksums. This format is
111 // suitable for serializing to disk.
112 public string ToStringWithHeader(VxSchemaChecksum sum)
114 byte[] md5 = MD5.Create().ComputeHash(text.ToUTF8());
116 return String.Format("!!SCHEMAMATIC {0} {1} \r\n{2}",
117 md5.ToHex().ToLower(), sum.GetSumString(), text);
120 public int CompareTo(object obj)
122 if (!(obj is VxSchemaElement))
123 throw new ArgumentException("object is not a VxSchemaElement");
125 VxSchemaElement other = (VxSchemaElement)obj;
126 return GetKey().CompareTo(other.GetKey());
129 public static string GetDbusSignature()
131 return "sssy";
135 // Represents an element of a table, such as a column or index.
136 // Each element has a type, such as "column", and a series of key,value
137 // parameters, such as ("name","MyColumn"), ("type","int"), etc.
138 internal class VxSchemaTableElement
140 public string elemtype;
141 // We can't use a Dictionary<string,string> because we might have repeated
142 // keys (such as two columns for an index).
143 public List<KeyValuePair<string,string>> parameters;
145 public VxSchemaTableElement(string type)
147 elemtype = type;
148 parameters = new List<KeyValuePair<string,string>>();
151 public void AddParam(string name, string val)
153 parameters.Add(new KeyValuePair<string,string>(name, val));
156 // Returns the first parameter found with the given name.
157 // Returns an empty string if none found.
158 public string GetParam(string name)
160 foreach (var kvp in parameters)
161 if (kvp.Key == name)
162 return kvp.Value;
163 return "";
166 // Returns a list of all parameters found with the given name.
167 // Returns an empty list if none found.
168 public List<string> GetParamList(string name)
170 List<string> results = new List<string>();
171 foreach (var kvp in parameters)
172 if (kvp.Key == name)
173 results.Add(kvp.Value);
174 return results;
177 public bool HasDefault()
179 return elemtype == "column" && GetParam("default").ne();
182 // Serializes to "elemtype: key1=value1,key2=value2" format.
183 public override string ToString()
185 StringBuilder sb = new StringBuilder();
186 bool empty = true;
187 sb.Append(elemtype + ": ");
188 foreach (var param in parameters)
190 if (!empty)
191 sb.Append(",");
193 sb.Append(param.Key + "=" + param.Value);
195 empty = false;
197 return sb.ToString();
200 // Returns a string uniquely identifying this element within the table.
201 // Generally includes the element's name, if it has one.
202 public string GetElemKey()
204 if (elemtype == "primary-key")
205 return elemtype;
206 else
207 return elemtype + ": " + GetParam("name");
211 internal class VxSchemaTable : VxSchemaElement,
212 IEnumerable<VxSchemaTableElement>
214 // A list of table elements, so we can maintain the original order
215 private List<VxSchemaTableElement> elems;
216 // A dictionary of table elements, so we can quickly check if we have
217 // an element.
218 private Dictionary<string, VxSchemaTableElement> elemdict;
220 public VxSchemaTable(string newname) :
221 base("Table", newname, null, false)
223 elems = new List<VxSchemaTableElement>();
224 elemdict = new Dictionary<string, VxSchemaTableElement>();
227 public VxSchemaTable(string newname, string newtext) :
228 base("Table", newname, null, false)
230 elems = new List<VxSchemaTableElement>();
231 elemdict = new Dictionary<string, VxSchemaTableElement>();
232 // Parse the new text
233 text = newtext;
236 public VxSchemaTable(VxSchemaElement elem) :
237 base("Table", elem.name, null, false)
239 elems = new List<VxSchemaTableElement>();
240 elemdict = new Dictionary<string, VxSchemaTableElement>();
241 // Parse the new text
242 text = elem.text;
245 public bool Contains(string elemkey)
247 return elemdict.ContainsKey(elemkey);
250 public VxSchemaTableElement this[string elemkey]
254 return elemdict[elemkey];
258 // Implement the IEnumerator interface - just punt to the list
259 IEnumerator IEnumerable.GetEnumerator()
261 return elems.GetEnumerator();
264 public IEnumerator<VxSchemaTableElement> GetEnumerator()
266 return elems.GetEnumerator();
269 public override string text
271 // Other schema elements just store their text verbatim.
272 // We parse it on input and recreate it on output in order to
273 // provide more sensible updating of tables in the database.
276 StringBuilder sb = new StringBuilder();
277 foreach (var elem in elems)
278 sb.Append(elem.ToString() + "\n");
279 return sb.ToString();
283 elems.Clear();
284 elemdict.Clear();
285 char[] equals = {'='};
286 char[] comma = {','};
287 foreach (string line in value.Split('\n'))
289 line.Trim();
290 if (line.Length == 0)
291 continue;
293 string typeseparator = ": ";
294 int index = line.IndexOf(typeseparator);
295 if (index < 0)
296 throw new ArgumentException
297 (wv.fmt("Malformed line in {0}: {1}", key, line));
298 string type = line.Remove(index);
299 string rest = line.Substring(index + typeseparator.Length);
301 var elem = new VxSchemaTableElement(type);
303 foreach (string kvstr in rest.Split(comma))
305 string[] kv = kvstr.Split(equals, 2);
306 if (kv.Length != 2)
307 throw new ArgumentException(wv.fmt(
308 "Invalid entry '{0}' in line '{1}'",
309 kvstr, line));
311 elem.parameters.Add(
312 new KeyValuePair<string,string>(kv[0], kv[1]));
314 Add(elem);
319 public string GetDefaultPKName()
321 return "PK_" + this.name;
324 public string GetDefaultDefaultName(string colname)
326 return wv.fmt("{0}_{1}_default", this.name, colname);
329 // Include any default constraints by, er, default.
330 public string ColumnToSql(VxSchemaTableElement elem)
332 return ColumnToSql(elem, true);
335 public string ColumnToSql(VxSchemaTableElement elem, bool include_default)
337 string colname = elem.GetParam("name");
338 string typename = elem.GetParam("type");
339 string lenstr = elem.GetParam("length");
340 string defval = elem.GetParam("default");
341 string nullstr = elem.GetParam("null");
342 string prec = elem.GetParam("precision");
343 string scale = elem.GetParam("scale");
344 string ident_seed = elem.GetParam("identity_seed");
345 string ident_incr = elem.GetParam("identity_incr");
347 string identstr = "";
348 if (ident_seed.ne() && ident_incr.ne())
349 identstr = wv.fmt(" IDENTITY ({0},{1})", ident_seed, ident_incr);
351 if (nullstr.e())
352 nullstr = "";
353 else if (nullstr == "0")
354 nullstr = " NOT NULL";
355 else
356 nullstr = " NULL";
358 if (lenstr.ne())
359 lenstr = " (" + lenstr + ")";
360 else if (prec.ne() && scale.ne())
361 lenstr = wv.fmt(" ({0},{1})", prec, scale);
363 if (include_default && defval.ne())
365 string defname = GetDefaultDefaultName(colname);
366 defval = " CONSTRAINT " + defname + " DEFAULT " + defval;
368 else
369 defval = "";
371 return wv.fmt("[{0}] [{1}]{2}{3}{4}{5}",
372 colname, typename, lenstr, defval, nullstr, identstr);
375 public string IndexToSql(VxSchemaTableElement elem)
377 List<string> idxcols = elem.GetParamList("column");
378 string idxname = elem.GetParam("name");
379 string unique = elem.GetParam("unique");
380 string clustered = elem.GetParam("clustered") == "1" ?
381 "CLUSTERED " : "";
383 if (unique != "" && unique != "0")
384 unique = "UNIQUE ";
385 else
386 unique = "";
388 return wv.fmt(
389 "CREATE {0}{1}INDEX [{2}] ON [{3}] \n\t({4});",
390 unique, clustered, idxname, this.name, idxcols.join(", "));
393 public string PrimaryKeyToSql(VxSchemaTableElement elem)
395 List<string> idxcols = elem.GetParamList("column");
396 string idxname = elem.GetParam("name");
397 string clustered = elem.GetParam("clustered") == "1" ?
398 " CLUSTERED" : " NONCLUSTERED";
400 if (idxname.e())
401 idxname = GetDefaultPKName();
403 return wv.fmt(
404 "ALTER TABLE [{0}] ADD CONSTRAINT [{1}] PRIMARY KEY{2}\n" +
405 "\t({3});\n\n",
406 this.name, idxname, clustered, idxcols.join(", "));
409 public override string ToSql()
411 List<string> cols = new List<string>();
412 List<string> indexes = new List<string>();
413 string pkey = "";
414 foreach (var elem in elems)
416 if (elem.elemtype == "column")
417 cols.Add(ColumnToSql(elem));
418 else if (elem.elemtype == "index")
419 indexes.Add(IndexToSql(elem));
420 else if (elem.elemtype == "primary-key")
422 if (pkey != "")
424 throw new VxBadSchemaException(
425 "Multiple primary key statements are not " +
426 "permitted in table definitions.\n" +
427 "Conflicting statement: " + elem.ToString() + "\n");
429 pkey = PrimaryKeyToSql(elem);
433 if (cols.Count == 0)
434 throw new VxBadSchemaException("No columns in schema.");
436 string table = String.Format("CREATE TABLE [{0}] (\n\t{1});\n\n{2}{3}\n",
437 name, cols.join(",\n\t"), pkey, indexes.join("\n"));
438 return table;
441 private void Add(VxSchemaTableElement elem)
443 elems.Add(elem);
445 string elemkey = elem.GetElemKey();
447 if (elemdict.ContainsKey(elemkey))
448 throw new VxBadSchemaException(wv.fmt("Duplicate table entry " +
449 "'{0}' found.", elemkey));
451 elemdict.Add(elemkey, elem);
454 public void AddColumn(string name, string type, int isnullable,
455 string len, string defval, string prec, string scale,
456 int isident, string ident_seed, string ident_incr)
458 var elem = new VxSchemaTableElement("column");
459 // FIXME: Put the table name here or not? Might be handy, but could
460 // get out of sync with e.g. filename or whatnot.
461 elem.AddParam("name", name);
462 elem.AddParam("type", type);
463 elem.AddParam("null", isnullable.ToString());
464 if (len.ne())
465 elem.AddParam("length", len);
466 if (defval.ne())
467 elem.AddParam("default", defval);
468 if (prec.ne())
469 elem.AddParam("precision", prec);
470 if (scale.ne())
471 elem.AddParam("scale", scale);
472 if (isident != 0)
474 elem.AddParam("identity_seed", ident_seed);
475 elem.AddParam("identity_incr", ident_incr);
477 Add(elem);
480 public void AddIndex(string name, int unique, int clustered,
481 params string[] columns)
483 WvLog log = new WvLog("AddIndex", WvLog.L.Debug4);
484 log.print("Adding index on {0}, name={1}, unique={2}, clustered={3},\n",
485 columns.join(","), name, unique, clustered);
486 var elem = new VxSchemaTableElement("index");
488 foreach (string col in columns)
489 elem.AddParam("column", col);
490 elem.AddParam("name", name);
491 elem.AddParam("unique", unique.ToString());
492 elem.AddParam("clustered", clustered.ToString());
494 Add(elem);
497 public void AddPrimaryKey(string name, int clustered,
498 params string[] columns)
500 WvLog log = new WvLog("AddPrimaryKey", WvLog.L.Debug4);
501 log.print("Adding primary key '{0}' on {1}, clustered={2}\n",
502 name, columns.join(","), clustered);
503 var elem = new VxSchemaTableElement("primary-key");
505 if (name.ne() && name != GetDefaultPKName())
506 elem.AddParam("name", name);
508 foreach (string col in columns)
509 elem.AddParam("column", col);
510 elem.AddParam("clustered", clustered.ToString());
512 Add(elem);
515 // Figure out what changed between oldtable and newtable.
516 // Returns any deleted elements first, followed by any modified or added
517 // elements in the same order they occur in newtable. Any returned
518 // elements scheduled for changing are from the new table.
519 public static List<KeyValuePair<VxSchemaTableElement, VxDiffType>> GetDiff(
520 VxSchemaTable oldtable, VxSchemaTable newtable)
522 WvLog log = new WvLog("SchemaTable GetDiff", WvLog.L.Debug4);
523 var diff = new List<KeyValuePair<VxSchemaTableElement, VxDiffType>>();
525 foreach (var elem in oldtable.elems)
527 string elemkey = elem.GetElemKey();
528 if (!newtable.Contains(elemkey))
530 log.print("Scheduling {0} for removal.\n", elemkey);
531 diff.Add(new KeyValuePair<VxSchemaTableElement, VxDiffType>(
532 oldtable[elemkey], VxDiffType.Remove));
535 foreach (var elem in newtable.elems)
537 string elemkey = elem.GetElemKey();
538 if (!oldtable.Contains(elemkey))
540 log.print("Scheduling {0} for addition.\n", elemkey);
541 diff.Add(new KeyValuePair<VxSchemaTableElement, VxDiffType>(
542 newtable[elemkey], VxDiffType.Add));
544 else if (elem.ToString() != oldtable[elemkey].ToString())
546 log.print("Scheduling {0} for change.\n", elemkey);
547 diff.Add(new KeyValuePair<VxSchemaTableElement, VxDiffType>(
548 newtable[elemkey], VxDiffType.Change));
552 return diff;
556 // The schema elements for a set of database elements
557 internal class VxSchema : Dictionary<string, VxSchemaElement>
559 public static ISchemaBackend create(string moniker)
561 ISchemaBackend sm = WvMoniker<ISchemaBackend>.create(moniker);
562 if (sm == null && Directory.Exists(moniker))
563 sm = WvMoniker<ISchemaBackend>.create("dir:" + moniker);
564 if (sm == null)
565 sm = WvMoniker<ISchemaBackend>.create("dbi:" + moniker);
566 if (sm == null)
567 throw new Exception
568 (wv.fmt("No moniker found for '{0}'", moniker));
569 return sm;
572 public VxSchema()
576 // Convenience method for making single-element schemas
577 public VxSchema(VxSchemaElement elem)
579 Add(elem.key, elem);
582 public VxSchema(VxSchema copy)
584 foreach (KeyValuePair<string,VxSchemaElement> p in copy)
585 this.Add(p.Key, VxSchemaElement.create(p.Value));
588 public VxSchema(IEnumerable<WvAutoCast> sch)
590 foreach (var row in sch)
592 VxSchemaElement elem = VxSchemaElement.create(row);
593 Add(elem.GetKey(), elem);
597 public void WriteSchema(WvDbusWriter writer)
599 writer.WriteArray(8, this, (w2, p) => {
600 p.Value.Write(w2);
604 // Returns only the elements of the schema that are affected by the diff.
605 // If an element is scheduled to be removed, clear its text field.
606 // Produces a VxSchema that, if sent to a schema backend's Put, will
607 // update the schema as indicated by the diff.
608 public VxSchema GetDiffElements(VxSchemaDiff diff)
610 VxSchema diffschema = new VxSchema();
611 foreach (KeyValuePair<string,VxDiffType> p in diff)
613 if (!this.ContainsKey(p.Key))
614 throw new ArgumentException("The provided diff does not " +
615 "match the schema: extra element '" +
616 (char)p.Value + " " + p.Key + "'");
617 if (p.Value == VxDiffType.Remove)
619 VxSchemaElement elem = VxSchemaElement.create(this[p.Key]);
620 elem.text = "";
621 diffschema[p.Key] = elem;
623 else if (p.Value == VxDiffType.Add || p.Value == VxDiffType.Change)
625 diffschema[p.Key] = VxSchemaElement.create(this[p.Key]);
628 return diffschema;
631 public void Add(string type, string name, string text, bool encrypted)
633 string key = GetKey(type, name, encrypted);
634 if (this.ContainsKey(key))
635 this[key].text += text;
636 else
637 this.Add(key, VxSchemaElement.create(type, name, text, encrypted));
640 public static string GetKey(string type, string name, bool encrypted)
642 string enc_str = encrypted ? "-Encrypted" : "";
643 return String.Format("{0}{1}/{2}", type, enc_str, name);
646 // ParseKey used to live here, but moved to VxSchemaChecksums.
647 public static void ParseKey(string key, out string type, out string name)
649 VxSchemaChecksums.ParseKey(key, out type, out name);
650 return;
653 public static string GetDbusSignature()
655 return String.Format("a({0})", VxSchemaElement.GetDbusSignature());
658 // Make dest look like source. Only copies the bits that need updating.
659 // Note: this is a slightly funny spot to put this method; it really
660 // belongs in ISchemaBackend, but you can't put methods in interfaces.
661 public static VxSchemaErrors CopySchema(ISchemaBackend source,
662 ISchemaBackend dest)
664 return VxSchema.CopySchema(source, dest, VxCopyOpts.None);
667 public static VxSchemaErrors CopySchema(ISchemaBackend source,
668 ISchemaBackend dest, VxCopyOpts opts)
670 WvLog log = new WvLog("CopySchema");
672 if ((opts & VxCopyOpts.ShowProgress) == 0)
673 log = new WvLog("CopySchema", WvLog.L.Debug5);
675 bool show_diff = (opts & VxCopyOpts.ShowDiff) != 0;
676 bool dry_run = (opts & VxCopyOpts.DryRun) != 0;
677 bool destructive = (opts & VxCopyOpts.Destructive) != 0;
679 log.print("Retrieving schema checksums from source.\n");
680 VxSchemaChecksums srcsums = source.GetChecksums();
682 log.print("Retrieving schema checksums from dest.\n");
683 VxSchemaChecksums destsums = dest.GetChecksums();
685 if (srcsums.Count == 0 && destsums.Count != 0)
687 log.print("Source index is empty! " +
688 "Refusing to delete entire database.\n");
689 return new VxSchemaErrors();
692 List<string> names = new List<string>();
694 log.print("Computing diff.\n");
695 VxSchemaDiff diff = new VxSchemaDiff(destsums, srcsums);
697 if (diff.Count == 0)
699 log.print("No changes.\n");
700 return new VxSchemaErrors();
703 if (show_diff)
705 log.print("Changes to apply:\n");
706 log.print(WvLog.L.Info, diff.ToString());
709 log.print("Parsing diff.\n");
710 List<string> to_drop = new List<string>();
711 foreach (KeyValuePair<string,VxDiffType> p in diff)
713 switch (p.Value)
715 case VxDiffType.Remove:
716 to_drop.Add(p.Key);
717 break;
718 case VxDiffType.Add:
719 case VxDiffType.Change:
720 names.Add(p.Key);
721 break;
725 log.print("Retrieving updated schema.\n");
726 VxSchema to_put = source.Get(names);
728 if (dry_run)
729 return new VxSchemaErrors();
731 VxSchemaErrors drop_errs = new VxSchemaErrors();
732 VxSchemaErrors put_errs = new VxSchemaErrors();
734 // We know at least one of to_drop and to_put must have something in
735 // it, otherwise the diff would have been empty.
737 if (to_drop.Count > 0)
739 log.print("Dropping deleted elements.\n");
740 drop_errs = dest.DropSchema(to_drop);
743 VxPutOpts putopts = VxPutOpts.None;
744 if (destructive)
745 putopts |= VxPutOpts.Destructive;
746 if (names.Count > 0)
748 log.print("Updating and adding elements.\n");
749 put_errs = dest.Put(to_put, srcsums, putopts);
752 // Combine the two sets of errors.
753 foreach (var kvp in drop_errs)
754 put_errs.Add(kvp.Key, kvp.Value);
756 return put_errs;