Updated the TODOs
[cerebrum.git] / generator.py
blob5ba604ad2ffdcd756ddd7e5746062296c262d96d
1 #Copyright (C) 2012 jaseg <s@jaseg.de>
3 #This program is free software; you can redistribute it and/or
4 #modify it under the terms of the GNU General Public License
5 #version 3 as published by the Free Software Foundation.
7 import subprocess
8 import os.path
9 import time
10 import random
11 from threading import Thread
12 import struct
13 from inspect import isfunction
14 from mako.template import Template
15 import binascii
16 import json
17 try:
18 import lzma
19 except:
20 import pylzma as lzma
21 import codecs
22 import unittest
23 """Automatic Cerebrum c code generator"""
25 # Code templates. Actually, this is the only device-dependent place in this whole
26 # file, and actually only a very few lines are device-dependent.
27 # FIXME: Break this stuff into a common "C code generator" which is included from
28 # here and from the msp code generator and which is feeded with the two or three
29 # device-dependent lines
31 autocode_header = """\
32 /* AUTOGENERATED CODE FOLLOWS!
33 * This file contains the code generated from the module templates as well as
34 * some glue logic. It is generated following the device config by "generate.py"
35 * in this very folder. Please refrain from modifying it, modify the templates
36 * and generation logic instead.
38 * Build version: ${version}, build date: ${builddate}
41 #include <string.h>
42 #include "autocode.h"
43 #include "comm.h"
44 #include "uart.h"
45 """
47 #This one contains the actual callback/init/loop magick.
48 autocode_footer = """
49 #include "config.h"
50 #if defined(__AVR__)
51 #include <avr/pgmspace.h>
52 #endif
54 void generic_getter_callback(const comm_callback_descriptor* cb, void* argbuf_end);
56 const comm_callback_descriptor comm_callbacks[] = {
57 % for (callback, argbuf, argbuf_len, id) in callbacks:
58 {${callback}, (void*)${argbuf}, ${argbuf_len}}, //${id}
59 % endfor
62 const uint16_t callback_count = (sizeof(comm_callbacks)/sizeof(comm_callback_descriptor)); //${len(callbacks)};
64 void init_auto(){
65 % for initfunc in init_functions:
66 ${initfunc}();
67 % endfor
70 void loop_auto(){
71 comm_loop();
72 % for loopfunc in loop_functions:
73 ${loopfunc}();
74 % endfor
77 void callback_get_descriptor_auto(const comm_callback_descriptor* cb, void* argbuf_end){
78 //FIXME
79 uart_putc(auto_config_descriptor_length >> 8);
80 uart_putc(auto_config_descriptor_length & 0xFF);
81 for(const char* i=auto_config_descriptor; i < auto_config_descriptor+auto_config_descriptor_length; i++){
82 #if defined(__AVR__)
83 uart_putc(pgm_read_byte(i));
84 #else
85 uart_putc(*i);
86 #endif
90 //Generic getter used for any readable parameters.
91 //Please note one curious thing: This callback can not only be used to read, but also to write a variable. The only
92 //difference between the setter and the getter of a variable is that the setter does not read the entire variable's
93 //contents aloud.
94 void generic_getter_callback(const comm_callback_descriptor* cb, void* argbuf_end){
95 //response length
96 uart_putc(cb->argbuf_len>>8);
97 uart_putc(cb->argbuf_len&0xFF);
98 //response
99 for(char* i=((char*)cb->argbuf); i<((char*)cb->argbuf)+cb->argbuf_len; i++){
100 uart_putc(*i);
106 config_c_template = """\
107 /* AUTOGENERATED CODE AHEAD!
108 * This file contains the device configuration in lzma-ed json-format. It is
109 * autogenerated by "generate.py" (which should be found in this folder).
111 #include "config.h"
112 #ifndef PROGMEM
113 #define PROGMEM
114 #endif
116 unsigned int auto_config_descriptor_length = ${desc_len};
117 char const auto_config_descriptor[] PROGMEM = {${desc}};
120 #FIXME possibly make a class out of this one
121 #FIXME I think the target parameter is not used anywhere. Remove?
122 def generate(desc, device, build_path, builddate, target = 'all', config_address=None):
123 members = desc["members"]
124 seqnum = 23 #module number (only used during build time to generate unique names)
125 current_id = 0
126 desc["builddate"] = str(builddate)
127 config_address = config_address or random.randint(0, 65534)
128 autocode = Template(autocode_header).render_unicode(version=desc["version"], builddate=builddate)
129 init_functions = []
130 loop_functions = []
131 callbacks = []
133 def register_callback(name, argbuf="global_argbuf", argbuf_len="ARGBUF_SIZE"):
134 nonlocal current_id
135 callbacks.append(("0" if name is None else "&"+name, argbuf, argbuf_len, current_id))
136 old_id = current_id
137 current_id += 1
138 return old_id
140 #Default callback number 0
141 register_callback("callback_get_descriptor_auto")
143 for mname, member in members.items():
144 mfile = member["type"]
145 mtype = mfile.replace('-', '_')
146 typepath = os.path.join(build_path, mfile + ".c.tp")
148 #CAUTION! These *will* exhibit strange behavior when called more than once!
149 def init_function():
150 fun = "init_{}_{}".format(mtype, seqnum)
151 init_functions.append(fun)
152 return fun
153 def loop_function():
154 fun = "loop_{}_{}".format(mtype, seqnum)
155 loop_functions.append(fun)
156 return fun
158 #module instance build config entries
159 properties = {}
160 functions = {}
162 #FIXME possibly swap the positions of ctype and fmt
163 def modulevar(name, ctype=None, fmt=None, array=False, callbacks=(0, 0)):
164 """Get the c name of a module variable and possibly register the variable with the code generator.
166 If only `name` is given, the autogenerated c name of the module variable will be returned.
168 If you provide `fmt`, virtual accessor methods for the variable will be registered and the variable will
169 be registered as a property in the build config (using the previously mentioned accessor methods).
170 In this context, "virtual" means that there will be callbacks in the callback list, but the setter will
171 not be backed by an actual function and the getter will just point to a global generic getter.
173 If you also provide `ctype` the accessors will also be generated.
175 `array` can be used to generated accessors for module variables that are arrays.
177 `callbacks` can be a tuple of one or two values. Each value corresponds to one callback. If the tuple contains
178 only one value, no setter will be generated and the variable will be marked read-only. A value of `None` prompts
179 the generation of the "default" accessor function. A value of `True` prompts the registration of an accessor
180 function of the form `callback_(get|set)_${modulevar(name)}` whose argument is stored in the module variable
181 buffer itself and which you must implement yourself. You may also specify a tuple of the form `(cbname, buf, buflen)`
182 where `cbname` is the name of your callback and `buf` and `buflen` are the argument buffer and argument buffer length,
183 respectively.
185 varname = "modvar_{}_{}_{}".format(mtype, seqnum, name)
186 if fmt is not None:
187 aval = 1
188 if array != False:
189 aval = array
191 def accessor_callback(desc, cbtype, defcb):
192 if desc == 0:
193 return register_callback(defcb, ("" if array else "&")+varname, "sizeof("+varname+")")
194 elif desc == 1:
195 return register_callback("callback_{}_{}".format(cbtype, varname), ("" if array else "&")+varname, "sizeof("+varname+")")
196 elif desc == 2:
197 return register_callback("callback_{}_{}".format(cbtype, varname), "global_argbuf", "ARGBUF_SIZE")
198 else:
199 cbname, buf, buflen = desc
200 if cbname is True:
201 cbname = "callback_{}_{}".format(cbtype, varname)
202 return register_callback(cbname, buf, buflen)
204 properties[name] = {
205 "size": struct.calcsize(fmt),
206 "id": accessor_callback(callbacks[0], 'get', 'generic_getter_callback'),
207 "fmt": fmt}
209 if callbacks[1] is not None:
210 accessor_callback(callbacks[1], 'set', None)
211 else:
212 #Save some space in the build config (that later gets burned into the µC's
213 #really small flash!) by only putting this here in case of read-only access
214 properties[name]["access"] = 'r'
216 if ctype is not None:
217 array_component = ""
218 if array == True:
219 array_component = "[]"
220 elif array:
221 array_component = "[{}]".format(array)
222 return "{} {}{}".format(ctype, varname, array_component)
223 else:
224 assert(ctype is None)
226 return varname
228 def module_callback(name, argformat="", retformat="", regname=None):
229 """Register a regular module callback.
231 I hereby officially discourage the (sole) use of this function since these callbacks or functions as they
232 appear at the Cerebrum level cannot be automatically mapped to snmp MIBs in any sensible manner. Thus, please
233 use accessors for everything if possible, even if it is stuff that you would not traditionally use them for.
234 For an example on how to generate and register custom accessor methods please see simple-io.c.tp .
237 cbname = 'callback_{}_{}_{}'.format(mtype, seqnum, name)
238 cbid = register_callback(regname or cbname)
239 func = { 'id': cbid }
240 #Save some space in the build config (that later gets burned into the µC's really small flash!)
241 if argformat is not '':
242 func['args'] = argformat
243 if retformat is not '':
244 func['returns'] = retformat
245 functions[name] = func
246 return cbname
248 #Flesh out the module template!
249 tp = Template(filename=typepath)
250 autocode += tp.render_unicode(
251 init_function=init_function,
252 loop_function=loop_function,
253 modulevar=modulevar,
254 setter=lambda x: 'callback_set_'+modulevar(x),
255 getter=lambda x: 'callback_get_'+modulevar(x),
256 module_callback=module_callback,
257 register_callback=register_callback,
258 member=member,
259 device=device)
261 #Save some space in the build config (that later gets burned into the µC's really small flash!)
262 if functions:
263 member['functions'] = functions
264 if properties:
265 member['properties'] = properties
267 #increment the module number
268 seqnum += 1
270 #finish the code generation and write the generated code to a file
271 autocode += Template(autocode_footer).render_unicode(init_functions=init_functions, loop_functions=loop_functions, callbacks=callbacks)
272 with open(os.path.join(build_path, 'autocode.c'), 'w') as f:
273 f.write(autocode)
274 #compress the build config and write it out
275 #Depending on whether you want to store the device config as plain text or lzma'ed plain text comment out one of the following lines
276 #The first byte is used as a magic here. The first byte of a JSON string will always be a '{'
277 config = b'#' + lzma.compress(bytes(json.JSONEncoder(separators=(',',':')).encode(desc), 'utf-8'))
278 #config = bytes(json.JSONEncoder(separators=(',',':')).encode(desc), 'utf-8')
279 with open(os.path.join(build_path, 'config.c'), 'w') as f:
280 f.write(Template(config_c_template).render_unicode(desc_len=len(config), desc=','.join(map(str, config))))
281 #compile the whole stuff
282 make_env = os.environ.copy()
283 make_env['MCU'] = device.get('mcu')
284 make_env['CLOCK'] = str(device.get('clock'))
285 make_env['CEREBRUM_BAUDRATE'] = str(device.get('cerebrum_baudrate'))
286 make_env['CONFIG_ADDRESS'] = str(config_address) #65535 is reserved as discovery address
287 subprocess.check_call(['/usr/bin/env', 'make', '--no-print-directory', '-C', build_path, 'clean', target], env=make_env)
289 return desc
291 def commit(device, build_path, args):
292 """Flash the newly generated firmware onto the device"""
293 make_env = os.environ.copy()
294 make_env['MCU'] = device.get('mcu')
295 make_env['PORT'] = args.port
296 make_env['PROGRAMMER'] = device.get('programmer')
297 make_env['PROGRAMMER_BAUDRATE'] = str(device.get('programmer_baudrate'))
298 subprocess.check_call(['/usr/bin/env', "make",'--no-print-directory', '-C', build_path, 'program'], env=make_env)
300 class TestBuild(unittest.TestCase):
302 def setUp(self):
303 pass
305 def test_basic_build(self):
306 generate({'members': {}, 'version': 0.17}, {'mcu': 'test'}, 'test', '2012-05-23 23:42:17', config_address=0x2342)
308 class TestCommStuff(unittest.TestCase):
310 def setUp(self):
311 generate({'members': {'test': {'type': 'test'}}, 'version': 0.17}, {'mcu': 'test'}, 'test', '2012-05-23 23:42:17', config_address=0x2342)
313 def new_test_process(self):
314 #spawn a new communication test process
315 p = subprocess.Popen([os.path.join(os.path.dirname(__file__), 'test', 'main')], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
317 #start a thread killing that process after a few seconds
318 def kill_subprocess():
319 time.sleep(5)
320 if p.poll() is None or p.returncode < 0:
321 p.terminate()
322 self.assert_(False, 'Communication test process terminated due to a timeout')
324 t = Thread(target=kill_subprocess)
325 t.daemon = True
326 t.start()
327 return (p, p.stdin, p.stdout, t)
329 def test_config_descriptor(self):
330 (p, stdin, stdout, t) = self.new_test_process();
332 stdin.write(b'\\#\x23\x42\x00\x00\x00\x00')
333 stdin.flush()
334 stdin.close()
336 (length,) = struct.unpack('>H', stdout.read(2))
337 #FIXME this fixed size comparision is *not* helpful.
338 #self.assertEqual(length, 227, 'Incorrect config descriptor length')
339 data = stdout.read(length)
340 #self.assertEqual(data, b']\x00\x00\x80\x00\x00=\x88\x8a\xc6\x94S\x90\x86\xa6c}%:\xbbAj\x14L\xd9\x1a\xae\x93n\r\x10\x83E1\xba]j\xdeG\xb1\xba\xa6[:\xa2\xb9\x8eR~#\xb9\x84%\xa0#q\x87\x17[\xd6\xcdA)J{\xab*\xf7\x96%\xff\xfa\x12g\x00', 'wrong config descriptor returned')
341 #Somehow, each time this is compiled, the json encode shuffles the fields of the object in another way. Thus it does not suffice to compare the plain strings.
342 #self.assert_(compareJSON(data, b'{"version":0.17,"builddate":"2012-05-23 23:42:17","members":{"test":{"functions":{"test_multipart":{"args":"65B","id":1},"check_test_buffer":{"id":4}},"type":"test","properties":{"test_buffer":{"size":65,"id":2,"fmt":"65B"}}}}}'), "The generated test program returns a wrong config descriptor: {}.".format(data))
343 #FIXME somehow, this commented-out device descriptor check fails randomly even though nothing is actually wrong.
345 def test_multipart_call(self):
346 (p, stdin, stdout, t) = self.new_test_process();
348 stdin.write(b'\\#\x23\x42\x00\x01\x00\x41AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA')
349 stdin.flush()
350 stdin.close()
352 #wait for test process to terminate. If everything else fails, the timeout thread will kill it.
353 p.wait()
354 self.assertEqual(p.returncode, 0, "The test process caught an error from the c code. Please watch stderr for details.")
356 def test_meta_multipart_call(self):
357 """Test whether the test function actually fails when given invalid data."""
358 (p, stdin, stdout, t) = self.new_test_process();
360 stdin.write(b'\\#\x23\x42\x00\x01\x00\x41AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAA')
361 stdin.flush()
362 stdin.close()
364 #wait for test process to terminate. If everything else fails, the timeout thread will kill it.
365 p.wait()
366 self.assertEqual(p.returncode, 1, "The test process did not catch an error it was supposed to catch from the c code. Please watch stderr for details.")
368 def test_multipart_call_long_args(self):
369 (p, stdin, stdout, t) = self.new_test_process();
371 stdin.write(b'\\#\x23\x42\x00\x05\x01\x01'+b'A'*257)
372 stdin.write(b'\\#\x23\x42\x00\x06\x00\x00')
373 stdin.flush()
374 stdin.close()
376 #wait for test process to terminate. If everything else fails, the timeout thread will kill it.
377 p.wait()
378 self.assertEqual(p.returncode, 0, "The test process caught an error from the c code. Please watch stderr for details.")
379 self.assertEqual(p.returncode, 0, "The test process caught an error from the c code. Please watch stderr for details.")
381 def test_meta_multipart_call_long_args(self):
382 """Test whether the test function actually fails when given invalid data."""
383 (p, stdin, stdout, t) = self.new_test_process();
385 stdin.write(b'\\#\x23\x42\x00\x05\x01\x01'+b'A'*128+b'B'+b'A'*128)
386 stdin.flush()
387 stdin.close()
389 #wait for test process to terminate. If everything else fails, the timeout thread will kill it.
390 p.wait()
391 self.assertEqual(p.returncode, 1, "The test process did not catch an error it was supposed to catch from the c code. Please watch stderr for details.")
393 def test_attribute_accessors_multipart(self):
394 (p, stdin, stdout, t) = self.new_test_process();
396 stdin.write(b'\\#\x23\x42\x00\x03\x01\x01'+b'A'*32+b'B'*32+b'C'*32+b'D'*32+b'E'*32+b'F'*32+b'G'*32+b'H'*32+b'I') # write some characters to test_buffer
397 stdin.write(b'\\#\x23\x42\x00\x04\x00\x00') # call check_test_buffer
398 stdin.flush()
399 stdin.close()
401 #wait for test process to terminate. If everything else fails, the timeout thread will kill it.
402 p.wait()
403 self.assertEqual(p.returncode, 0, "The test process caught an error from the c code. Please watch stderr for details.")
405 def test_meta_attribute_accessors_multipart(self):
406 (p, stdin, stdout, t) = self.new_test_process();
408 stdin.write(b'\\#\x23\x42\x00\x03\x01\x01'+b'A'*33+b'B'*31+b'C'*32+b'D'*32+b'E'*32+b'F'*32+b'G'*32+b'H'*32+b'I') # write some characters to test_buffer
409 stdin.write(b'\\#\x23\x42\x00\x04\x00\x00') # call check_test_buffer
410 stdin.flush()
411 stdin.close()
413 #wait for test process to terminate. If everything else fails, the timeout thread will kill it.
414 p.wait()
415 self.assertEqual(p.returncode, 1, "The test process did not catch an error it was supposed to catch from the c code. Please watch stderr for details.")