src/scriptvm/scanner.l

/*
 * Copyright (c) 2014-2017 Christian Schoenebeck and Andreas Persson
 *
 * http://www.linuxsampler.org
 *
 * This file is part of LinuxSampler and released under the same terms.
 * See README file for details.
 */

/* Token scanner for NKSP real-time instrument script language. */

%{

#include "parser_shared.h"
#include <math.h>
// reentrant scanner data context
#define YY_EXTRA_TYPE ParserContext*
// set line number each time a token is recognized
#define YY_USER_ACTION                  \
{                                       \
    yylloc->first_line = yylineno;      \
    yylloc->last_line  = yylineno;      \
    yylloc->first_column = yycolumn;    \
    yycolumn += yyleng;                 \
    yylloc->last_column = yycolumn - 1; \
    /*printf("lex: line '%s'\n", yytext);*/  \
}
// custom (f)lex input for reading from std::istream object
#define YY_INPUT(buf,result,max_size)   \
{                                       \
    char c = yyextra->is->get();        \
    if (yyextra->is->eof())             \
        result = YY_NULL;               \
    else {                              \
        buf[0] = c;                     \
        result = 1;                     \
    }                                   \
}

static void scanner_error(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* err) {
    context->addErr(locp->first_line, locp->last_line, locp->first_column, locp->last_column, err);
}

static void scanner_warning(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* txt) {
    context->addWrn(locp->first_line, locp->last_line, locp->first_column, locp->last_column, txt);
}

#define SCANNER_ERR(txt)  scanner_error(yylloc, yyextra, txt)
#define SCANNER_WRN(txt)  scanner_warning(yylloc, yyextra, txt)

// shut up warning that 'register' keyword is deprecated as of C++11
#if defined(__cplusplus) && __cplusplus >= 201103L
# define register
#endif

using namespace LinuxSampler;

static int countNewLineChars(const char* txt) {
    int n = 0;
    for (int i = 0; txt[i]; ++i)
        if (txt[i] == '\n') ++n;
    return n;
}

static int countCharsPastLastNewLine(const char* txt) {
    const int n = (int)strlen(txt);
    for (int i = n - 1; i >= 0; --i)
        if (txt[i] == '\n')
            return n - i - 1;
    return n;
}

#define processLocation() { \
    const int nl = countNewLineChars(yytext); \
    yylineno += nl; \
    if (nl) yycolumn = countCharsPastLastNewLine(yytext); \
}

// if compiled for debugging, throw an exception instead of exiting on fatal
// lexer errors (so the debugger may pause with the appropriate back trace)
#if DEBUG
# include <stdexcept>
# define YY_FATAL_ERROR(msg) throw std::runtime_error(msg)
#endif

%}

/* use Flex's built-in support for line numbers
   (disabled, because it seems to be unreliable, so we are using our own
   tracking code in the respective scanner rules below) */
/*%option yylineno*/
/* generate a reentrant safe scanner */
%option reentrant
/* avoid symbol collision with other (i.e. future) scanner symbols */
%option prefix="InstrScript_"
/* bison-bridge adds an argument yylval to yylex, and bison-locations adds an
   argument code yylloc for location tracking. */
%option bison-bridge
%option bison-locations
/* yywrap() would be called at EOF, we don't need it */
%option noyywrap
/* enable functions yy_push_state(), yy_pop_state(), yy_top_state() */
%option stack

/* inclusive scanner conditions */
%s PREPROC_BODY_USE
/* exclusive scanner conditions */
%x PREPROC_SET_COND PREPROC_RESET_COND PREPROC_IF PREPROC_IF_NOT PREPROC_BODY_EAT PREPROC_PRE_BODY_USE PREPROC_PRE_BODY_EAT

DIGIT    [0-9]
ID       [a-zA-Z0-9_]+

%%

\"[^"]*\" {
    yylval->sValue = strdup(yytext + 1);
    yylval->sValue[strlen(yylval->sValue) - 1] = '\0';
    return STRING;
}

{DIGIT}+ {
    yylval->iValue = atoi(yytext);
    return INTEGER;
}

 /* there is currently no support for floating point numbers in NKSP yet */
 /*{DIGIT}+"."{DIGIT}* {
    printf("A float: %s (%g)\n", yytext, atof(yytext));
 }*/


 /* Preprocessor statement:  SET_CONDITION(name) */

<*>"SET_CONDITION"[ \t]*"(" {
    //printf("SET_CONDITION\n");
    yy_push_state(PREPROC_SET_COND, yyscanner);
}
<PREPROC_SET_COND>{ID} {
    //printf("preproc set condition '%s'\n", yytext);
    bool success = yyextra->setPreprocessorCondition(yytext);
    if (!success) {
        SCANNER_WRN((String("Preprocessor: Condition '") +
                    yytext + "' is already set.").c_str());
    }
}
<PREPROC_SET_COND>[ \t]*")" {
    //printf("End of PREPROC_SET_COND\n");
    yy_pop_state(yyscanner);
}


 /* Preprocessor statement:  RESET_CONDITION(name) */

<*>"RESET_CONDITION"[ \t]*"(" {
    //printf("RESET_CONDITION\n");
    yy_push_state(PREPROC_RESET_COND, yyscanner);
}
<PREPROC_RESET_COND>{ID} {
    //printf("preproc reset condition '%s'\n", yytext);
    bool success = yyextra->resetPreprocessorCondition(yytext);
    if (!success) {
        SCANNER_ERR((String("Preprocessor: could not reset condition '") +
                    yytext + "' (either not set or a built-in condition).").c_str());
    }
}
<PREPROC_RESET_COND>[ \t]*")" {
    //printf("End of RESET_CONDITION\n");
    yy_pop_state(yyscanner);
}


 /* Preprocessor conditional statements:

    USE_CODE_IF(name)
      ...
    END_USE_CODE

    and:

    USE_CODE_IF_NOT(name)
      ...
    END_USE_CODE    
 */

<*>"USE_CODE_IF"[ \t]*"(" {
    //printf("{%s}\n", yytext);
    yy_push_state(PREPROC_IF, yyscanner);
}
<PREPROC_BODY_EAT>"USE_CODE_IF"[ \t]*"("{ID}")" {
    //printf("[EAT{%s}\n", yytext);
    yy_push_state(PREPROC_BODY_EAT, yyscanner);
}
<*>"USE_CODE_IF_NOT"[ \t]*"(" {
    //printf("USE_CODE_IF_NOT\n");
    yy_push_state(PREPROC_IF_NOT, yyscanner);
}
<PREPROC_BODY_EAT>"USE_CODE_IF_NOT"[ \t]*"("{ID}")" {
    //printf("[EAT{%s}\n", yytext);
    yy_push_state(PREPROC_BODY_EAT, yyscanner);
}
<PREPROC_IF>{ID} {
    //printf("preproc use code if '%s'\n", yytext);
    yy_pop_state(yyscanner);
    if (yyextra->isPreprocessorConditionSet(yytext))
        yy_push_state(PREPROC_PRE_BODY_USE, yyscanner);
    else
        yy_push_state(PREPROC_PRE_BODY_EAT, yyscanner);
}
<PREPROC_IF_NOT>{ID} {
    //printf("preproc use code if not '%s'\n", yytext);
    yy_pop_state(yyscanner);
    if (!yyextra->isPreprocessorConditionSet(yytext))
        yy_push_state(PREPROC_PRE_BODY_USE, yyscanner);
    else
        yy_push_state(PREPROC_PRE_BODY_EAT, yyscanner);
}
<PREPROC_PRE_BODY_USE>[ \t]*")" {
    yy_pop_state(yyscanner);
    yy_push_state(PREPROC_BODY_USE, yyscanner);
}
<PREPROC_PRE_BODY_EAT>[ \t]*")" {
    //printf("PREPROCESSOR EAT : {%s}\n", yytext);
    yy_pop_state(yyscanner);
    yy_push_state(PREPROC_BODY_EAT, yyscanner);
}
<PREPROC_BODY_EAT,PREPROC_BODY_USE>"END_USE_CODE" {
    //printf("-->END_USE_CODE\n");
    yy_pop_state(yyscanner);
}
<PREPROC_BODY_EAT>[ \t\r\n]* { /* eat up code block filtered out by preprocessor */
    //printf("PREPROCESSOR EAT2 : {%s}\n", yytext);
    processLocation();
}
<PREPROC_BODY_EAT>.* { /* eat up code block filtered out by preprocessor */
    //printf("PREPROCESSOR EAT3 : {%s}\n", yytext);
    yyextra->addPreprocessorComment(yylloc->first_line, yylloc->last_line,
                                    yylloc->first_column+1, yylloc->last_column+1);
}


 /* Language keywords */

"on" return ON;
"end" return END;
"note" return NOTE;
"init" return INIT;
"declare" return DECLARE;
"while" return WHILE;
"if" return IF;
".or." return BITWISE_OR;
"or" return OR;
"release" return RELEASE;
".and." return BITWISE_AND;
"and" return AND;
".not." return BITWISE_NOT;
"not" return NOT;
"else" return ELSE;
"controller" return CONTROLLER;
"case" return CASE;
"select" return SELECT;
"to" return TO;
"<=" return LE;
">=" return GE;
"const" return CONST_; // note: "CONST" is already defined for C/C++ compilers on Windows by default
"polyphonic" return POLYPHONIC;
"mod" return MOD;
"function" return FUNCTION;
"call" return CALL;
"synchronized" return SYNCHRONIZED;

[&,()[\]<>=*+#/-] {
    return *yytext;
}

("$"|"@"|"%"){ID} {
    yylval->sValue = strdup(yytext);
    return VARIABLE;
}

{ID} {
    yylval->sValue = strdup(yytext);
    return IDENTIFIER;
}

":=" return ASSIGNMENT;

\n+ {
     yylineno += countNewLineChars(yytext);
     yycolumn = 0;
    //printf("lex: new line %d\n", yylineno, yytext);
    //return LF;
}

"{"[^}]*"}" { /* eat up comments */
    processLocation();
}

[ \t\r]+ /* eat up whitespace */

. {
    printf( "Unrecognized character: '%s' (line %d, column %d)\n", yytext, yylineno, yycolumn);
    return UNKNOWN_CHAR;
}

%%

namespace LinuxSampler {

void ParserContext::createScanner(std::istream* is) {
   if (scanner) destroyScanner();
   this->is = is;
   yylex_init(&scanner);
   yyset_extra(this, scanner);
}

void ParserContext::destroyScanner() {
   if (!scanner) return;
   yylex_destroy(scanner);
   scanner = NULL;
}

} // namespace LinuxSampler
1	schoenebeck	2581	/*
2	schoenebeck	3259	* Copyright (c) 2014-2017 Christian Schoenebeck and Andreas Persson
3	schoenebeck	2581	*
4			* http://www.linuxsampler.org
5			*
6			* This file is part of LinuxSampler and released under the same terms.
7			* See README file for details.
8			*/
9
10	schoenebeck	2888	/* Token scanner for NKSP real-time instrument script language. */
11	schoenebeck	2581
12			%{
13
14			#include "parser_shared.h"
15			#include <math.h>
16			// reentrant scanner data context
17			#define YY_EXTRA_TYPE ParserContext*
18			// set line number each time a token is recognized
19			#define YY_USER_ACTION \
20			{ \
21			yylloc->first_line = yylineno; \
22			yylloc->last_line = yylineno; \
23	schoenebeck	2888	yylloc->first_column = yycolumn; \
24			yycolumn += yyleng; \
25			yylloc->last_column = yycolumn - 1; \
26	schoenebeck	2581	/printf("lex: line '%s'\n", yytext);/ \
27			}
28			// custom (f)lex input for reading from std::istream object
29			#define YY_INPUT(buf,result,max_size) \
30			{ \
31			char c = yyextra->is->get(); \
32			if (yyextra->is->eof()) \
33			result = YY_NULL; \
34			else { \
35			buf[0] = c; \
36			result = 1; \
37			} \
38			}
39
40			static void scanner_error(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* err) {
41	schoenebeck	2889	context->addErr(locp->first_line, locp->last_line, locp->first_column, locp->last_column, err);
42	schoenebeck	2581	}
43
44			static void scanner_warning(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* txt) {
45	schoenebeck	2889	context->addWrn(locp->first_line, locp->last_line, locp->first_column, locp->last_column, txt);
46	schoenebeck	2581	}
47
48			#define SCANNER_ERR(txt) scanner_error(yylloc, yyextra, txt)
49			#define SCANNER_WRN(txt) scanner_warning(yylloc, yyextra, txt)
50
51	schoenebeck	3054	// shut up warning that 'register' keyword is deprecated as of C++11
52			#if defined(__cplusplus) && __cplusplus >= 201103L
53			# define register
54			#endif
55
56	schoenebeck	2581	using namespace LinuxSampler;
57
58	schoenebeck	2888	static int countNewLineChars(const char* txt) {
59			int n = 0;
60			for (int i = 0; txt[i]; ++i)
61			if (txt[i] == '\n') ++n;
62			return n;
63			}
64
65			static int countCharsPastLastNewLine(const char* txt) {
66	schoenebeck	3054	const int n = (int)strlen(txt);
67	schoenebeck	2888	for (int i = n - 1; i >= 0; --i)
68			if (txt[i] == '\n')
69			return n - i - 1;
70			return n;
71			}
72
73			#define processLocation() { \
74			const int nl = countNewLineChars(yytext); \
75			yylineno += nl; \
76			if (nl) yycolumn = countCharsPastLastNewLine(yytext); \
77			}
78
79	schoenebeck	3308	// if compiled for debugging, throw an exception instead of exiting on fatal
80			// lexer errors (so the debugger may pause with the appropriate back trace)
81			#if DEBUG
82			# include <stdexcept>
83			# define YY_FATAL_ERROR(msg) throw std::runtime_error(msg)
84			#endif
85
86	schoenebeck	2581	%}
87
88	schoenebeck	2888	/* use Flex's built-in support for line numbers
89			(disabled, because it seems to be unreliable, so we are using our own
90			tracking code in the respective scanner rules below) */
91			/%option yylineno/
92	schoenebeck	2581	/* generate a reentrant safe scanner */
93			%option reentrant
94			/* avoid symbol collision with other (i.e. future) scanner symbols */
95			%option prefix="InstrScript_"
96			/* bison-bridge adds an argument yylval to yylex, and bison-locations adds an
97			argument code yylloc for location tracking. */
98			%option bison-bridge
99			%option bison-locations
100			/* yywrap() would be called at EOF, we don't need it */
101			%option noyywrap
102			/* enable functions yy_push_state(), yy_pop_state(), yy_top_state() */
103			%option stack
104
105			/* inclusive scanner conditions */
106			%s PREPROC_BODY_USE
107			/* exclusive scanner conditions */
108			%x PREPROC_SET_COND PREPROC_RESET_COND PREPROC_IF PREPROC_IF_NOT PREPROC_BODY_EAT PREPROC_PRE_BODY_USE PREPROC_PRE_BODY_EAT
109
110			DIGIT [0-9]
111	schoenebeck	2935	ID [a-zA-Z0-9_]+
112	schoenebeck	2581
113			%%
114
115			\"[^"]*\" {
116			yylval->sValue = strdup(yytext + 1);
117			yylval->sValue[strlen(yylval->sValue) - 1] = '\0';
118			return STRING;
119			}
120
121			{DIGIT}+ {
122			yylval->iValue = atoi(yytext);
123			return INTEGER;
124			}
125
126	schoenebeck	2935	/* there is currently no support for floating point numbers in NKSP yet */
127			/{DIGIT}+"."{DIGIT} {
128	schoenebeck	2581	printf("A float: %s (%g)\n", yytext, atof(yytext));
129	schoenebeck	2935	}*/
130	schoenebeck	2581
131
132			/* Preprocessor statement: SET_CONDITION(name) */
133
134			<>"SET_CONDITION"[ \t]"(" {
135	schoenebeck	2588	//printf("SET_CONDITION\n");
136	schoenebeck	2581	yy_push_state(PREPROC_SET_COND, yyscanner);
137			}
138			<PREPROC_SET_COND>{ID} {
139	schoenebeck	2588	//printf("preproc set condition '%s'\n", yytext);
140	schoenebeck	2581	bool success = yyextra->setPreprocessorCondition(yytext);
141			if (!success) {
142			SCANNER_WRN((String("Preprocessor: Condition '") +
143			yytext + "' is already set.").c_str());
144			}
145			}
146			<PREPROC_SET_COND>[ \t]*")" {
147	schoenebeck	2588	//printf("End of PREPROC_SET_COND\n");
148	schoenebeck	2581	yy_pop_state(yyscanner);
149			}
150
151
152			/* Preprocessor statement: RESET_CONDITION(name) */
153
154			<>"RESET_CONDITION"[ \t]"(" {
155	schoenebeck	2588	//printf("RESET_CONDITION\n");
156	schoenebeck	2581	yy_push_state(PREPROC_RESET_COND, yyscanner);
157			}
158			<PREPROC_RESET_COND>{ID} {
159	schoenebeck	2588	//printf("preproc reset condition '%s'\n", yytext);
160	schoenebeck	2581	bool success = yyextra->resetPreprocessorCondition(yytext);
161			if (!success) {
162			SCANNER_ERR((String("Preprocessor: could not reset condition '") +
163			yytext + "' (either not set or a built-in condition).").c_str());
164			}
165			}
166			<PREPROC_RESET_COND>[ \t]*")" {
167	schoenebeck	2588	//printf("End of RESET_CONDITION\n");
168	schoenebeck	2581	yy_pop_state(yyscanner);
169			}
170
171
172			/* Preprocessor conditional statements:
173
174			USE_CODE_IF(name)
175			...
176			END_USE_CODE
177
178			and:
179
180			USE_CODE_IF_NOT(name)
181			...
182			END_USE_CODE
183			*/
184
185			<>"USE_CODE_IF"[ \t]"(" {
186	schoenebeck	3308	//printf("{%s}\n", yytext);
187	schoenebeck	2581	yy_push_state(PREPROC_IF, yyscanner);
188			}
189	schoenebeck	3308	<PREPROC_BODY_EAT>"USE_CODE_IF"[ \t]*"("{ID}")" {
190			//printf("[EAT{%s}\n", yytext);
191			yy_push_state(PREPROC_BODY_EAT, yyscanner);
192			}
193	schoenebeck	2581	<>"USE_CODE_IF_NOT"[ \t]"(" {
194	schoenebeck	2588	//printf("USE_CODE_IF_NOT\n");
195	schoenebeck	2581	yy_push_state(PREPROC_IF_NOT, yyscanner);
196			}
197	schoenebeck	3308	<PREPROC_BODY_EAT>"USE_CODE_IF_NOT"[ \t]*"("{ID}")" {
198			//printf("[EAT{%s}\n", yytext);
199			yy_push_state(PREPROC_BODY_EAT, yyscanner);
200			}
201	schoenebeck	2581	<PREPROC_IF>{ID} {
202	schoenebeck	2588	//printf("preproc use code if '%s'\n", yytext);
203	schoenebeck	2581	yy_pop_state(yyscanner);
204			if (yyextra->isPreprocessorConditionSet(yytext))
205			yy_push_state(PREPROC_PRE_BODY_USE, yyscanner);
206			else
207			yy_push_state(PREPROC_PRE_BODY_EAT, yyscanner);
208			}
209			<PREPROC_IF_NOT>{ID} {
210	schoenebeck	2588	//printf("preproc use code if not '%s'\n", yytext);
211	schoenebeck	2581	yy_pop_state(yyscanner);
212			if (!yyextra->isPreprocessorConditionSet(yytext))
213			yy_push_state(PREPROC_PRE_BODY_USE, yyscanner);
214			else
215			yy_push_state(PREPROC_PRE_BODY_EAT, yyscanner);
216			}
217			<PREPROC_PRE_BODY_USE>[ \t]*")" {
218			yy_pop_state(yyscanner);
219			yy_push_state(PREPROC_BODY_USE, yyscanner);
220			}
221			<PREPROC_PRE_BODY_EAT>[ \t]*")" {
222	schoenebeck	3308	//printf("PREPROCESSOR EAT : {%s}\n", yytext);
223	schoenebeck	2581	yy_pop_state(yyscanner);
224			yy_push_state(PREPROC_BODY_EAT, yyscanner);
225			}
226	schoenebeck	3308	<PREPROC_BODY_EAT,PREPROC_BODY_USE>"END_USE_CODE" {
227	schoenebeck	2588	//printf("-->END_USE_CODE\n");
228	schoenebeck	2581	yy_pop_state(yyscanner);
229			}
230	schoenebeck	2888	<PREPROC_BODY_EAT>[ \t\r\n]* { /* eat up code block filtered out by preprocessor */
231	schoenebeck	3308	//printf("PREPROCESSOR EAT2 : {%s}\n", yytext);
232	schoenebeck	2888	processLocation();
233			}
234			<PREPROC_BODY_EAT>.* { /* eat up code block filtered out by preprocessor */
235	schoenebeck	3308	//printf("PREPROCESSOR EAT3 : {%s}\n", yytext);
236	schoenebeck	3285	yyextra->addPreprocessorComment(yylloc->first_line, yylloc->last_line,
237			yylloc->first_column+1, yylloc->last_column+1);
238	schoenebeck	2888	}
239	schoenebeck	2581
240
241			/* Language keywords */
242
243			"on" return ON;
244			"end" return END;
245			"note" return NOTE;
246			"init" return INIT;
247			"declare" return DECLARE;
248			"while" return WHILE;
249			"if" return IF;
250	schoenebeck	2935	".or." return BITWISE_OR;
251	schoenebeck	2581	"or" return OR;
252			"release" return RELEASE;
253	schoenebeck	2935	".and." return BITWISE_AND;
254	schoenebeck	2581	"and" return AND;
255	schoenebeck	2935	".not." return BITWISE_NOT;
256	schoenebeck	2581	"not" return NOT;
257			"else" return ELSE;
258			"controller" return CONTROLLER;
259			"case" return CASE;
260			"select" return SELECT;
261			"to" return TO;
262			"<=" return LE;
263			">=" return GE;
264	schoenebeck	2585	"const" return CONST_; // note: "CONST" is already defined for C/C++ compilers on Windows by default
265	schoenebeck	2581	"polyphonic" return POLYPHONIC;
266			"mod" return MOD;
267	schoenebeck	2951	"function" return FUNCTION;
268			"call" return CALL;
269	schoenebeck	3260	"synchronized" return SYNCHRONIZED;
270	schoenebeck	2581
271	schoenebeck	2935	[&,()[\]<>=*+#/-] {
272			return *yytext;
273	schoenebeck	2581	}
274
275	schoenebeck	2935	("$"\|"@"\|"%"){ID} {
276	schoenebeck	2581	yylval->sValue = strdup(yytext);
277			return VARIABLE;
278			}
279
280			{ID} {
281			yylval->sValue = strdup(yytext);
282			return IDENTIFIER;
283			}
284
285			":=" return ASSIGNMENT;
286
287			\n+ {
288	schoenebeck	2888	yylineno += countNewLineChars(yytext);
289			yycolumn = 0;
290	schoenebeck	2581	//printf("lex: new line %d\n", yylineno, yytext);
291			//return LF;
292			}
293
294	schoenebeck	2888	"{"[^}]"}" { / eat up comments */
295			processLocation();
296			}
297	schoenebeck	2581
298			[ \t\r]+ /* eat up whitespace */
299
300	schoenebeck	3259	. {
301			printf( "Unrecognized character: '%s' (line %d, column %d)\n", yytext, yylineno, yycolumn);
302			return UNKNOWN_CHAR;
303			}
304	schoenebeck	2581
305			%%
306
307			namespace LinuxSampler {
308
309			void ParserContext::createScanner(std::istream* is) {
310			if (scanner) destroyScanner();
311			this->is = is;
312			yylex_init(&scanner);
313			yyset_extra(this, scanner);
314			}
315
316			void ParserContext::destroyScanner() {
317			if (!scanner) return;
318			yylex_destroy(scanner);
319			scanner = NULL;
320			}
321
322			} // namespace LinuxSampler