python/samba/ms_forest_updates_markdown.py

   1 # Create forest updates ldif from Github markdown
   2 #
   3 # Each update is converted to an ldif then gets written to a corresponding
   4 # .LDF output file or stored in a dictionary.
   5 #
   6 # Only add updates can generally be applied.
   7 #
   8 # Copyright (C) Andrew Bartlett <abartlet@samba.org> 2017
   9 #
  10 # This program is free software; you can redistribute it and/or modify
  11 # it under the terms of the GNU General Public License as published by
  12 # the Free Software Foundation; either version 3 of the License, or
  13 # (at your option) any later version.
  14 #
  15 # This program is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 # GNU General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU General Public License
  21 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  22
  23 from __future__ import print_function
  24 """Generate LDIF from Github documentation."""
  25
  26 import re
  27 import os
  28 import markdown
  29 import xml.etree.ElementTree as ET
  30 from samba.compat import get_string
  31
  32
  33 # Display specifier updates or otherwise (ignored in forest_update.py)
  34 def noop(description, attributes, sd):
  35     return (None, None, [], None)
  36
  37
  38 # ACE addition updates (ignored in forest_update.py)
  39 def parse_grant(description, attributes, sd):
  40     return ('modify', None, [], sd if sd.lower() != 'n/a' else None)
  41
  42
  43 # Addition of new objects to the directory (most are applied in forest_update.py)
  44 def parse_add(description, attributes, sd):
  45     dn = extract_dn(description)
  46     return ('add', dn, extract_attrib(dn, attributes), sd if sd.lower() != 'n/a' else None)
  47
  48
  49 # Set of a particular attribute (ignored in forest_update.py)
  50 def parse_set(description, attributes, sd):
  51     return ('modify', extract_dn_or_none(description),
  52             extract_replace_attrib(attributes),
  53             sd if sd.lower() != 'n/a' else None)
  54
  55
  56 # Set of a particular ACE (ignored in forest_update.py)
  57 # The general issue is that the list of DNs must be generated dynamically
  58 def parse_ace(description, attributes, sd):
  59
  60     def extract_dn_ace(text):
  61         if 'Sam-Domain' in text:
  62             return ('${DOMAIN_DN}', 'CN=Sam-Domain,${SCHEMA_DN}')
  63         elif 'Domain-DNS' in text:
  64             return ('${...}', 'CN=Domain-DNS,${SCHEMA_DN}')
  65
  66         return None
  67
  68     return [('modify', extract_dn_ace(description)[0],
  69              ['replace: nTSecurityDescriptor',
  70               'nTSecurityDescriptor: ${DOMAIN_SCHEMA_SD}%s' % sd], None),
  71             ('modify', extract_dn_ace(description)[1],
  72              ['replace: defaultSecurityDescriptor',
  73               'defaultSecurityDescriptor: ${OLD_SAMBA_SD}%s' % sd], None)]
  74
  75
  76 # We are really only interested in 'Created' items
  77 operation_map = {
  78     # modify
  79     'Granting': parse_grant,
  80     # add
  81     'Created': parse_add,
  82     # modify
  83     'Set': parse_set,
  84     # modify
  85     'Added ACE': parse_ace,
  86     # modify
  87     'Updated': parse_set,
  88     # unknown
  89     'Call': noop
  90 }
  91
  92
  93 def extract_dn(text):
  94     """
  95     Extract a DN from the textual description
  96     :param text:
  97     :return: DN in string form
  98     """
  99     text = text.replace(' in the Schema partition.', ',${SCHEMA_DN}')
 100     text = text.replace(' in the Configuration partition.', ',${CONFIG_DN}')
 101     dn = re.search('([CDO][NCU]=.*?,)*([CDO][NCU]=.*)', text).group(0)
 102
 103     # This should probably be also fixed upstream
 104     if dn == 'CN=ad://ext/AuthenticationSilo,CN=Claim Types,CN=Claims Configuration,CN=Services':
 105         return 'CN=ad://ext/AuthenticationSilo,CN=Claim Types,CN=Claims Configuration,CN=Services,${CONFIG_DN}'
 106
 107     return dn
 108
 109
 110 def extract_dn_or_none(text):
 111     """
 112     Same as above, but returns None if it doesn't work
 113     :param text:
 114     :return: DN or None
 115     """
 116     try:
 117         return extract_dn(text)
 118     except:
 119         return None
 120
 121
 122 def save_ldif(filename, answers, out_folder):
 123     """
 124     Save ldif to disk for each updates
 125     :param filename: filename use ([OPERATION NUM]-{GUID}.ldif)
 126     :param answers: array of tuples generated with earlier functions
 127     :param out_folder: folder to prepend
 128     """
 129     path = os.path.join(out_folder, filename)
 130     with open(path, 'w') as ldif:
 131         for answer in answers:
 132             change, dn, attrib, sd = answer
 133             ldif.write('dn: %s\n' % dn)
 134             ldif.write('changetype: %s\n' % change)
 135             if len(attrib) > 0:
 136                 ldif.write('\n'.join(attrib) + '\n')
 137             if sd is not None:
 138                 ldif.write('nTSecurityDescriptor: D:%s\n' % sd)
 139             ldif.write('-\n\n')
 140
 141
 142 def save_array(guid, answers, out_dict):
 143     """
 144     Save ldif to an output dictionary
 145     :param guid: GUID to store
 146     :param answers: array of tuples generated with earlier functions
 147     :param out_dict: output dictionary
 148     """
 149     ldif = ''
 150     for answer in answers:
 151         change, dn, attrib, sd = answer
 152         ldif += 'dn: %s\n' % dn
 153         ldif += 'changetype: %s\n' % change
 154         if len(attrib) > 0:
 155             ldif += '\n'.join(attrib) + '\n'
 156         if sd is not None:
 157             ldif += 'nTSecurityDescriptor: D:%s\n' % sd
 158         ldif += '-\n\n'
 159
 160     out_dict[guid] = ldif
 161
 162
 163 def extract_attrib(dn, attributes):
 164     """
 165     Extract the attributes as an array from the attributes column
 166     :param dn: parsed from markdown
 167     :param attributes: from markdown
 168     :return: attribute array (ldif-type format)
 169     """
 170     attrib = [x.lstrip('- ') for x in attributes.split('-   ') if x.lower() != 'n/a' and x != '']
 171     attrib = [x.replace(': True', ': TRUE') if x.endswith(': True') else x for x in attrib]
 172     attrib = [x.replace(': False', ': FALSE') if x.endswith(': False') else x for x in attrib]
 173     # We only have one such value, we may as well skip them all consistently
 174     attrib = [x for x in attrib if not x.lower().startswith('msds-claimpossiblevalues')]
 175
 176     return attrib
 177
 178
 179 def extract_replace_attrib(attributes):
 180     """
 181     Extract the attributes as an array from the attributes column
 182     (for replace)
 183     :param attributes: from markdown
 184     :return: attribute array (ldif-type format)
 185     """
 186     lines = [x.lstrip('- ') for x in attributes.split('-   ') if x.lower() != 'n/a' and x != '']
 187     lines = [('replace: %s' % line.split(':')[0], line) for line in lines]
 188     lines = [line for pair in lines for line in pair]
 189     return lines
 190
 191
 192 def innertext(tag):
 193     return (tag.text or '') + \
 194         ''.join(innertext(e) for e in tag) + \
 195         (tag.tail or '')
 196
 197
 198 def read_ms_markdown(in_file, out_folder=None, out_dict={}):
 199     """
 200     Read Github documentation to produce forest wide udpates
 201     :param in_file: Forest-Wide-Updates.md
 202     :param out_folder: output folder
 203     :param out_dict: output dictionary
 204     """
 205
 206     with open(in_file) as update_file:
 207         # There is a hidden ClaimPossibleValues in this md file
 208         html = markdown.markdown(re.sub(r'CN=<forest root domain.*?>',
 209                                         '${FOREST_ROOT_DOMAIN}',
 210                                         update_file.read()),
 211                                  output_format='xhtml')
 212
 213     html = html.replace('CN=Schema,%ws', '${SCHEMA_DN}')
 214
 215     tree = ET.fromstring('<root>' + html + '</root>')
 216
 217     for node in tree:
 218         if node.text and node.text.startswith('|Operation'):
 219             # Strip first and last |
 220             updates = [x[1:len(x) - 1].split('|') for x in
 221                        get_string(ET.tostring(node, method='text')).splitlines()]
 222             for update in updates[2:]:
 223                 output = re.match('Operation (\d+): {(.*)}', update[0])
 224                 if output:
 225                     # print output.group(1), output.group(2)
 226                     guid = output.group(2)
 227                     filename = "%s-{%s}.ldif" % (output.group(1).zfill(4), guid)
 228
 229                 found = False
 230
 231                 if update[3].startswith('Created') or update[1].startswith('Added ACE'):
 232                     # Trigger the security descriptor code
 233                     # Reduce info to just the security descriptor
 234                     update[3] = update[3].split(':')[-1]
 235
 236                     result = parse_ace(update[1], update[2], update[3])
 237
 238                     if filename and out_folder is not None:
 239                         save_ldif(filename, result, out_folder)
 240                     else:
 241                         save_array(guid, result, out_dict)
 242
 243                     continue
 244
 245                 for operation in operation_map:
 246                     if update[1].startswith(operation):
 247                         found = True
 248
 249                         result = operation_map[operation](update[1], update[2], update[3])
 250
 251                         if filename and out_folder is not None:
 252                             save_ldif(filename, [result], out_folder)
 253                         else:
 254                             save_array(guid, [result], out_dict)
 255
 256                         break
 257
 258                 if not found:
 259                     raise Exception(update)
 260
 261             # print ET.tostring(node, method='text')
 262
 263
 264 if __name__ == '__main__':
 265     import sys
 266
 267     out_folder = ''
 268
 269     if len(sys.argv) == 0:
 270         print("Usage: %s <Forest-Wide-Updates.md> [<output folder>]" % (sys.argv[0]), file=sys.stderr)
 271         sys.exit(1)
 272
 273     in_file = sys.argv[1]
 274     if len(sys.argv) > 2:
 275         out_folder = sys.argv[2]
 276
 277     read_ms_markdown(in_file, out_folder)