/[svn]/linuxsampler/trunk/src/scriptvm/scanner.l
ViewVC logotype

Contents of /linuxsampler/trunk/src/scriptvm/scanner.l

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3729 - (show annotations) (download)
Fri Jan 31 10:57:53 2020 UTC (4 years, 1 month ago) by schoenebeck
File size: 12568 byte(s)
NKSP parser: track code locations also by raw byte position and length.

* NKSP VM API: Added member variables 'firstByte' and 'lengthBytes' to
  struct 'CodeBlock'.

* NKSP Editor API: Added methods firstByte() and lengthBytes() to
  class 'VMSourceToken'.

* NKSP VM language parser: track all positions also by raw byte offset
  and length (along to the already existing tracking by line/column).

* NKSP editor syntax highlighting scanner: track all positions also by
  raw byte offset and length (along to the already existing tracking by
  line/column).

* Bumped version (2.1.1.svn45).

1 /*
2 * Copyright (c) 2014-2017 Christian Schoenebeck and Andreas Persson
3 *
4 * http://www.linuxsampler.org
5 *
6 * This file is part of LinuxSampler and released under the same terms.
7 * See README file for details.
8 */
9
10 /* Token scanner for NKSP real-time instrument script language. */
11
12 %{
13
14 #include "parser_shared.h"
15 #include <math.h>
16 // reentrant scanner data context
17 #define YY_EXTRA_TYPE ParserContext*
18 // set line number each time a token is recognized
19 #define YY_USER_ACTION \
20 { \
21 yylloc->first_line = yylineno; \
22 yylloc->last_line = yylineno; \
23 yylloc->first_column = yycolumn; \
24 yylloc->first_byte = yyextra->nbytes; \
25 yylloc->length_bytes = (int) yyleng; \
26 yycolumn += yyleng; \
27 yyextra->nbytes += (int) yyleng; \
28 yylloc->last_column = yycolumn - 1; \
29 /*printf("lex: line '%s'\n", yytext);*/ \
30 }
31 // custom (f)lex input for reading from std::istream object
32 #define YY_INPUT(buf,result,max_size) \
33 { \
34 char c = yyextra->is->get(); \
35 if (yyextra->is->eof()) \
36 result = YY_NULL; \
37 else { \
38 buf[0] = c; \
39 result = 1; \
40 } \
41 }
42
43 static void scanner_error(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* err) {
44 context->addErr(locp->first_line, locp->last_line, locp->first_column,
45 locp->last_column, locp->first_byte, locp->length_bytes,
46 err);
47 }
48
49 static void scanner_warning(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* txt) {
50 context->addWrn(locp->first_line, locp->last_line, locp->first_column,
51 locp->last_column, locp->first_byte, locp->length_bytes,
52 txt);
53 }
54
55 #define SCANNER_ERR(txt) scanner_error(yylloc, yyextra, txt)
56 #define SCANNER_WRN(txt) scanner_warning(yylloc, yyextra, txt)
57
58 // shut up warning that 'register' keyword is deprecated as of C++11
59 #if defined(__cplusplus) && __cplusplus >= 201103L
60 # define register
61 #endif
62
63 using namespace LinuxSampler;
64
65 static int countNewLineChars(const char* txt) {
66 int n = 0;
67 for (int i = 0; txt[i]; ++i)
68 if (txt[i] == '\n') ++n;
69 return n;
70 }
71
72 static int countCharsPastLastNewLine(const char* txt) {
73 const int n = (int)strlen(txt);
74 for (int i = n - 1; i >= 0; --i)
75 if (txt[i] == '\n')
76 return n - i - 1;
77 return n;
78 }
79
80 #define processLocation() { \
81 const int nl = countNewLineChars(yytext); \
82 yylineno += nl; \
83 if (nl) yycolumn = countCharsPastLastNewLine(yytext); \
84 }
85
86 // if compiled for debugging, throw an exception instead of exiting on fatal
87 // lexer errors (so the debugger may pause with the appropriate back trace)
88 #if DEBUG
89 # include <stdexcept>
90 # define YY_FATAL_ERROR(msg) throw std::runtime_error(msg)
91 #endif
92
93 %}
94
95 /* use Flex's built-in support for line numbers
96 (disabled, because it seems to be unreliable, so we are using our own
97 tracking code in the respective scanner rules below) */
98 /*%option yylineno*/
99 /* generate a reentrant safe scanner */
100 %option reentrant
101 /* avoid symbol collision with other (i.e. future) scanner symbols */
102 %option prefix="InstrScript_"
103 /* bison-bridge adds an argument yylval to yylex, and bison-locations adds an
104 argument code yylloc for location tracking. */
105 %option bison-bridge
106 %option bison-locations
107 /* yywrap() would be called at EOF, we don't need it */
108 %option noyywrap
109 /* enable functions yy_push_state(), yy_pop_state(), yy_top_state() */
110 %option stack
111
112 /* inclusive scanner conditions */
113 %s PREPROC_BODY_USE
114 /* exclusive scanner conditions */
115 %x PREPROC_SET_COND PREPROC_RESET_COND PREPROC_IF PREPROC_IF_NOT PREPROC_BODY_EAT PREPROC_PRE_BODY_USE PREPROC_PRE_BODY_EAT
116
117 DIGIT [0-9]
118 ID [a-zA-Z][a-zA-Z0-9_]*
119 METRIC (k|h|(da)|d|c|m|u)
120 UNIT (s|(Hz)|B)
121
122 %%
123
124 \"[^"]*\" {
125 yylval->sValue = strdup(yytext + 1);
126 yylval->sValue[strlen(yylval->sValue) - 1] = '\0';
127 return STRING;
128 }
129
130 {DIGIT}+ {
131 if (sizeof(vmint) < 8)
132 yylval->iValue = atoi(yytext);
133 else
134 yylval->iValue = atoll(yytext);
135 return INTEGER;
136 }
137
138 {DIGIT}+"."{DIGIT}+ {
139 yylval->fValue = atof(yytext);
140 return REAL;
141 }
142
143 {DIGIT}+({METRIC}{1,2}|({METRIC}{0,2}{UNIT}?)) {
144 int pos = 0;
145
146 // parse number portion
147 vmint value = 0;
148 for (; yytext[pos] >= '0' && yytext[pos] <= '9'; ++pos) {
149 value *= 10;
150 value += yytext[pos] - '0';
151 }
152 yylval->iUnitValue.iValue = value;
153
154 // parse metric prefix portion
155 for (int i = 0; i < 2; ++i, ++pos) {
156 switch (yytext[pos]) {
157 case 'k': yylval->iUnitValue.prefix[i] = VM_KILO; continue;
158 case 'h': yylval->iUnitValue.prefix[i] = VM_HECTO; continue;
159 case 'c': yylval->iUnitValue.prefix[i] = VM_CENTI; continue;
160 case 'm': yylval->iUnitValue.prefix[i] = VM_MILLI; continue;
161 case 'u': yylval->iUnitValue.prefix[i] = VM_MICRO; continue;
162 case 'd':
163 if (yytext[pos+1] == 'a') {
164 yylval->iUnitValue.prefix[i] = VM_DECA;
165 ++pos;
166 } else {
167 yylval->iUnitValue.prefix[i] = VM_DECI;
168 }
169 continue;
170 default:
171 yylval->iUnitValue.prefix[i] = VM_NO_PREFIX;
172 goto parseIntStdUnit;
173 }
174 }
175
176 parseIntStdUnit:
177
178 // parse standard measurement unit
179 switch (yytext[pos]) {
180 case 's': yylval->iUnitValue.unit = VM_SECOND; break;
181 case 'H': yylval->iUnitValue.unit = VM_HERTZ; break;
182 case 'B': yylval->iUnitValue.unit = VM_BEL; break;
183 default: yylval->iUnitValue.unit = VM_NO_UNIT; break;
184 }
185
186 return INTEGER_UNIT;
187 }
188
189 {DIGIT}+"."{DIGIT}+({METRIC}{1,2}|({METRIC}{0,2}{UNIT}?)) {
190 int pos = 0;
191
192 // parse number portion
193 for (; (yytext[pos] >= '0' && yytext[pos] <= '9') || yytext[pos] == '.'; ++pos) {
194 }
195 {
196 const char tmp = yytext[pos];
197 yytext[pos] = 0; // mark temporary end of string
198 yylval->fUnitValue.fValue = atof(yytext);
199 yytext[pos] = tmp; // restore
200 }
201
202 // parse metric prefix portion
203 for (int i = 0; i < 2; ++i, ++pos) {
204 switch (yytext[pos]) {
205 case 'k': yylval->fUnitValue.prefix[i] = VM_KILO; continue;
206 case 'h': yylval->fUnitValue.prefix[i] = VM_HECTO; continue;
207 case 'c': yylval->fUnitValue.prefix[i] = VM_CENTI; continue;
208 case 'm': yylval->fUnitValue.prefix[i] = VM_MILLI; continue;
209 case 'u': yylval->fUnitValue.prefix[i] = VM_MICRO; continue;
210 case 'd':
211 if (yytext[pos+1] == 'a') {
212 yylval->fUnitValue.prefix[i] = VM_DECA;
213 ++pos;
214 } else {
215 yylval->fUnitValue.prefix[i] = VM_DECI;
216 }
217 continue;
218 default:
219 yylval->fUnitValue.prefix[i] = VM_NO_PREFIX;
220 goto parseRealStdUnit;
221 }
222 }
223
224 parseRealStdUnit:
225
226 // parse standard measurement unit
227 switch (yytext[pos]) {
228 case 's': yylval->fUnitValue.unit = VM_SECOND; break;
229 case 'H': yylval->fUnitValue.unit = VM_HERTZ; break;
230 case 'B': yylval->fUnitValue.unit = VM_BEL; break;
231 default: yylval->fUnitValue.unit = VM_NO_UNIT; break;
232 }
233
234 return REAL_UNIT;
235 }
236
237
238 /* Preprocessor statement: SET_CONDITION(name) */
239
240 <*>"SET_CONDITION"[ \t]*"(" {
241 //printf("SET_CONDITION\n");
242 yy_push_state(PREPROC_SET_COND, yyscanner);
243 }
244 <PREPROC_SET_COND>{ID} {
245 //printf("preproc set condition '%s'\n", yytext);
246 bool success = yyextra->setPreprocessorCondition(yytext);
247 if (!success) {
248 SCANNER_WRN((String("Preprocessor: Condition '") +
249 yytext + "' is already set.").c_str());
250 }
251 }
252 <PREPROC_SET_COND>[ \t]*")" {
253 //printf("End of PREPROC_SET_COND\n");
254 yy_pop_state(yyscanner);
255 }
256
257
258 /* Preprocessor statement: RESET_CONDITION(name) */
259
260 <*>"RESET_CONDITION"[ \t]*"(" {
261 //printf("RESET_CONDITION\n");
262 yy_push_state(PREPROC_RESET_COND, yyscanner);
263 }
264 <PREPROC_RESET_COND>{ID} {
265 //printf("preproc reset condition '%s'\n", yytext);
266 bool success = yyextra->resetPreprocessorCondition(yytext);
267 if (!success) {
268 SCANNER_ERR((String("Preprocessor: could not reset condition '") +
269 yytext + "' (either not set or a built-in condition).").c_str());
270 }
271 }
272 <PREPROC_RESET_COND>[ \t]*")" {
273 //printf("End of RESET_CONDITION\n");
274 yy_pop_state(yyscanner);
275 }
276
277
278 /* Preprocessor conditional statements:
279
280 USE_CODE_IF(name)
281 ...
282 END_USE_CODE
283
284 and:
285
286 USE_CODE_IF_NOT(name)
287 ...
288 END_USE_CODE
289 */
290
291 <*>"USE_CODE_IF"[ \t]*"(" {
292 //printf("{%s}\n", yytext);
293 yy_push_state(PREPROC_IF, yyscanner);
294 }
295 <PREPROC_BODY_EAT>"USE_CODE_IF"[ \t]*"("{ID}")" {
296 //printf("[EAT{%s}\n", yytext);
297 yy_push_state(PREPROC_BODY_EAT, yyscanner);
298 }
299 <*>"USE_CODE_IF_NOT"[ \t]*"(" {
300 //printf("USE_CODE_IF_NOT\n");
301 yy_push_state(PREPROC_IF_NOT, yyscanner);
302 }
303 <PREPROC_BODY_EAT>"USE_CODE_IF_NOT"[ \t]*"("{ID}")" {
304 //printf("[EAT{%s}\n", yytext);
305 yy_push_state(PREPROC_BODY_EAT, yyscanner);
306 }
307 <PREPROC_IF>{ID} {
308 //printf("preproc use code if '%s'\n", yytext);
309 yy_pop_state(yyscanner);
310 if (yyextra->isPreprocessorConditionSet(yytext))
311 yy_push_state(PREPROC_PRE_BODY_USE, yyscanner);
312 else
313 yy_push_state(PREPROC_PRE_BODY_EAT, yyscanner);
314 }
315 <PREPROC_IF_NOT>{ID} {
316 //printf("preproc use code if not '%s'\n", yytext);
317 yy_pop_state(yyscanner);
318 if (!yyextra->isPreprocessorConditionSet(yytext))
319 yy_push_state(PREPROC_PRE_BODY_USE, yyscanner);
320 else
321 yy_push_state(PREPROC_PRE_BODY_EAT, yyscanner);
322 }
323 <PREPROC_PRE_BODY_USE>[ \t]*")" {
324 yy_pop_state(yyscanner);
325 yy_push_state(PREPROC_BODY_USE, yyscanner);
326 }
327 <PREPROC_PRE_BODY_EAT>[ \t]*")" {
328 //printf("PREPROCESSOR EAT : {%s}\n", yytext);
329 yy_pop_state(yyscanner);
330 yy_push_state(PREPROC_BODY_EAT, yyscanner);
331 }
332 <PREPROC_BODY_EAT,PREPROC_BODY_USE>"END_USE_CODE" {
333 //printf("-->END_USE_CODE\n");
334 yy_pop_state(yyscanner);
335 }
336 <PREPROC_BODY_EAT>[ \t\r\n]* { /* eat up code block filtered out by preprocessor */
337 //printf("PREPROCESSOR EAT2 : {%s}\n", yytext);
338 processLocation();
339 }
340 <PREPROC_BODY_EAT>.* { /* eat up code block filtered out by preprocessor */
341 //printf("PREPROCESSOR EAT3 : {%s}\n", yytext);
342 yyextra->addPreprocessorComment(yylloc->first_line, yylloc->last_line,
343 yylloc->first_column+1, yylloc->last_column+1,
344 yylloc->first_byte, yylloc->length_bytes);
345 }
346
347
348 /* Language keywords */
349
350 "on" return ON;
351 "end" return END;
352 "note" return NOTE;
353 "init" return INIT;
354 "declare" return DECLARE;
355 "while" return WHILE;
356 "if" return IF;
357 ".or." return BITWISE_OR;
358 "or" return OR;
359 "release" return RELEASE;
360 ".and." return BITWISE_AND;
361 "and" return AND;
362 ".not." return BITWISE_NOT;
363 "not" return NOT;
364 "else" return ELSE;
365 "controller" return CONTROLLER;
366 "rpn" return RPN;
367 "nrpn" return NRPN;
368 "case" return CASE;
369 "select" return SELECT;
370 "to" return TO;
371 "<=" return LE;
372 ">=" return GE;
373 "const" return CONST_; // note: "CONST" is already defined for C/C++ compilers on Windows by default
374 "polyphonic" return POLYPHONIC;
375 "mod" return MOD;
376 "function" return FUNCTION;
377 "call" return CALL;
378 "synchronized" return SYNCHRONIZED;
379
380 [&,!()[\]<>=*+#\/-] {
381 return *yytext;
382 }
383
384 ("$"|"@"|"%"|"~"|"?"){ID} {
385 yylval->sValue = strdup(yytext);
386 return VARIABLE;
387 }
388
389 {ID} {
390 yylval->sValue = strdup(yytext);
391 return IDENTIFIER;
392 }
393
394 ":=" return ASSIGNMENT;
395
396 \n+ {
397 yylineno += countNewLineChars(yytext);
398 yycolumn = 0;
399 //printf("lex: new line %d\n", yylineno, yytext);
400 //return LF;
401 }
402
403 "{"[^}]*"}" { /* eat up comments */
404 processLocation();
405 }
406
407 [ \t\r]+ /* eat up whitespace */
408
409 . {
410 printf( "Unrecognized character: '%s' (line %d, column %d)\n", yytext, yylineno, yycolumn);
411 return UNKNOWN_CHAR;
412 }
413
414 %%
415
416 namespace LinuxSampler {
417
418 void ParserContext::createScanner(std::istream* is) {
419 if (scanner) destroyScanner();
420 this->is = is;
421 yylex_init(&scanner);
422 yyset_extra(this, scanner);
423 }
424
425 void ParserContext::destroyScanner() {
426 if (!scanner) return;
427 yylex_destroy(scanner);
428 scanner = NULL;
429 }
430
431 } // namespace LinuxSampler

  ViewVC Help
Powered by ViewVC