1 # LexGen.py - implemented 2002 by Neil Hodgson neilh@scintilla.org
2 # Released to the public domain.
4 # Regenerate the Scintilla and SciTE source files that list
5 # all the lexers and all the properties files.
6 # Should be run whenever a new lexer is added or removed.
7 # Requires Python 2.4 or later
8 # Most files are regenerated in place with templates stored in comments.
9 # The VS .NET project file is generated into a different file as the
10 # VS .NET environment will not retain comments when modifying the file.
11 # The files are copied to a string apart from sections between a
12 # ++Autogenerated comment and a --Autogenerated comment which is
13 # generated by the CopyWithInsertion function. After the whole
14 # string is instantiated, it is compared with the target file and
15 # if different the file is rewritten.
16 # Does not regenerate the Visual C++ 6 project files but does the VS .NET
28 if sys
.platform
== "win32":
31 # Yes, LF is the native EOL even on Mac OS X. CR is just for
32 # Mac OS <=9 (a.k.a. "Mac Classic")
35 # Automatically generated sections contain start and end comments,
36 # a definition line and the results.
37 # The results are replaced by regenerating based on the definition line.
38 # The definition line is a comment prefix followed by "**".
39 # If there is a digit after the ** then this indicates which list to use
40 # and the digit and next character are not part of the definition
41 # Backslash is used as an escape within the definition line.
42 # The part between \( and \) is repeated for each item in the list.
43 # \* is replaced by each list item. \t, and \n are tab and newline.
44 def CopyWithInsertion(input, commentPrefix
, retainDefs
, eolType
, *lists
):
48 for line
in input.splitlines(0):
49 isStartGenerated
= line
.startswith(commentPrefix
+ "++Autogenerated")
50 if copying
and not isStartGenerated
:
57 elif not copying
and line
.startswith(commentPrefix
+ "**"):
60 definition
= line
[len(commentPrefix
+ "**"):]
62 if definition
[0] in string
.digits
:
63 listid
= int(definition
[:1])
64 definition
= definition
[2:]
65 # Hide double slashes as a control character
66 definition
= definition
.replace("\\\\", "\001")
67 # Do some normal C style transforms
68 definition
= definition
.replace("\\n", "\n")
69 definition
= definition
.replace("\\t", "\t")
70 # Get the doubled backslashes back as single backslashes
71 definition
= definition
.replace("\001", "\\")
72 startRepeat
= definition
.find("\\(")
73 endRepeat
= definition
.find("\\)")
74 intro
= definition
[:startRepeat
]
76 if intro
.endswith("\n"):
81 middle
= definition
[startRepeat
+2:endRepeat
]
82 for i
in lists
[listid
]:
83 item
= middle
.replace("\\*", i
)
84 if pos
and (pos
+ len(item
) >= 80):
89 if item
.endswith("\n"):
91 outro
= definition
[endRepeat
+2:]
93 out
= out
.replace("\n", eolType
) # correct EOLs in generated content
95 elif line
.startswith(commentPrefix
+ "--Autogenerated"):
99 output
= [line
.rstrip(" \t") for line
in output
] # trim trailing whitespace
100 return eolType
.join(output
) + eolType
102 def UpdateFile(filename
, updated
):
103 """ If the file is different to updated then copy updated
104 into the file else leave alone so CVS and make don't treat
107 infile
= open(filename
, "rb")
108 except IOError: # File is not there yet
109 out
= open(filename
, "wb")
112 print "New", filename
114 original
= infile
.read()
116 if updated
!= original
:
118 out
= open(filename
, "wb")
121 print "Changed", filename
123 #~ print "Unchanged", filename
125 def Generate(inpath
, outpath
, commentPrefix
, eolType
, *lists
):
126 """Generate 'outpath' from 'inpath'.
128 "eolType" indicates the type of EOLs to use in the generated
129 file. It should be one of following constants: LF, CRLF,
132 #print "generate '%s' -> '%s' (comment prefix: %r, eols: %r)"\
133 # % (inpath, outpath, commentPrefix, eolType)
135 infile
= open(inpath
, "r")
137 print "Can not open", inpath
139 original
= infile
.read()
141 updated
= CopyWithInsertion(original
, commentPrefix
,
142 inpath
== outpath
, eolType
, *lists
)
143 UpdateFile(outpath
, updated
)
145 def Regenerate(filename
, commentPrefix
, eolType
, *lists
):
146 """Regenerate the given file.
148 "eolType" indicates the type of EOLs to use in the generated
149 file. It should be one of following constants: LF, CRLF,
152 Generate(filename
, filename
, commentPrefix
, eolType
, *lists
)
154 def FindModules(lexFile
):
157 for l
in f
.readlines():
158 if l
.startswith("LexerModule"):
159 l
= l
.replace("(", " ")
160 modules
.append(l
.split()[1])
163 knownIrregularProperties
= [
165 "styling.within.preprocessor",
166 "tab.timmy.whinge.level",
167 "asp.default.language",
168 "html.tags.case.sensitive",
171 "sql.backslash.escapes",
176 def FindProperties(lexFile
):
179 for l
in f
.readlines():
180 if "GetProperty" in l
:
182 if not l
.startswith("//"): # Drop comments
183 propertyName
= l
.split("\"")[1]
184 if propertyName
.lower() == propertyName
:
185 # Only allow lower case property names
186 if propertyName
in knownIrregularProperties
or \
187 propertyName
.startswith("fold.") or \
188 propertyName
.startswith("lexer."):
189 properties
.add(propertyName
)
193 return cmp(a
.lower(), b
.lower())
198 # Find all the lexer source code files
199 lexFilePaths
= glob
.glob(root
+ "scintilla/src/Lex*.cxx")
200 lexFiles
= [os
.path
.basename(f
)[:-4] for f
in lexFilePaths
]
203 lexerProperties
= set()
204 for lexFile
in lexFilePaths
:
205 lexerModules
.extend(FindModules(lexFile
))
206 lexerProperties
.update(FindProperties(lexFile
))
207 lexerModules
.sort(ciCompare
)
208 lexerProperties
.remove("fold.comment.python")
209 lexerProperties
= list(lexerProperties
)
210 lexerProperties
.sort(ciCompare
)
212 # Find all the SciTE properties files
213 otherProps
= ["abbrev.properties", "Embedded.properties", "SciTEGlobal.properties", "SciTE.properties"]
214 if os
.path
.exists(root
+ "scite"):
215 propFilePaths
= glob
.glob(root
+ "scite/src/*.properties")
216 propFiles
= [os
.path
.basename(f
) for f
in propFilePaths
if os
.path
.basename(f
) not in otherProps
]
217 propFiles
.sort(ciCompare
)
220 Regenerate(root
+ "scintilla/src/KeyWords.cxx", "//", NATIVE
, lexerModules
)
221 Regenerate(root
+ "scintilla/win32/makefile", "#", NATIVE
, lexFiles
)
222 Regenerate(root
+ "scintilla/win32/scintilla.mak", "#", NATIVE
, lexFiles
)
223 Regenerate(root
+ "scintilla/win32/scintilla_vc6.mak", "#", NATIVE
, lexFiles
)
224 # Use Unix EOLs for gtk Makefiles so they work for Linux users when
225 # extracted from the Scintilla source ZIP (typically created on
227 Regenerate(root
+ "scintilla/gtk/makefile", "#", LF
, lexFiles
)
228 Regenerate(root
+ "scintilla/gtk/scintilla.mak", "#", NATIVE
, lexFiles
)
229 Regenerate(root
+ "scintilla/macosx/makefile", "#", LF
, lexFiles
)
230 if os
.path
.exists(root
+ "scite"):
231 Regenerate(root
+ "scite/win32/makefile", "#", NATIVE
, lexFiles
, propFiles
)
232 Regenerate(root
+ "scite/win32/scite.mak", "#", NATIVE
, lexFiles
, propFiles
)
233 Regenerate(root
+ "scite/src/SciTEProps.cxx", "//", NATIVE
, lexerProperties
)
234 Generate(root
+ "scite/boundscheck/vcproj.gen",
235 root
+ "scite/boundscheck/SciTE.vcproj", "#", NATIVE
, lexFiles
)