src/scriptvm/scanner.l

/*
 * Copyright (c) 2014 Christian Schoenebeck and Andreas Persson
 *
 * http://www.linuxsampler.org
 *
 * This file is part of LinuxSampler and released under the same terms.
 * See README file for details.
 */

/* Token scanner for instrument script language. */

/* FIXME: line numbers (i.e. on error/warning messages) are incorrect, because
   the current grammar does not process new line characters. \n is currently
   filtered out in this lexer. Due to this, the parser will sometimes read
   several lines before it matches the next complete grammar rule, causing the
   incorrect lines in the error/warning messages. */

%{

#include "parser_shared.h"
#include <math.h>
// reentrant scanner data context
#define YY_EXTRA_TYPE ParserContext*
// set line number each time a token is recognized
#define YY_USER_ACTION                  \
{                                       \
    yylloc->first_line = yylineno;      \
    yylloc->last_line  = yylineno;      \
    /* first_column = TODO */           \
    /* last_column  = TODO */           \
    /*printf("lex: line '%s'\n", yytext);*/  \
}
// custom (f)lex input for reading from std::istream object
#define YY_INPUT(buf,result,max_size)   \
{                                       \
    char c = yyextra->is->get();        \
    if (yyextra->is->eof())             \
        result = YY_NULL;               \
    else {                              \
        buf[0] = c;                     \
        result = 1;                     \
    }                                   \
}

static void scanner_error(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* err) {
    context->addErr(locp->first_line, err);
}

static void scanner_warning(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* txt) {
    context->addWrn(locp->first_line, txt);
}

#define SCANNER_ERR(txt)  scanner_error(yylloc, yyextra, txt)
#define SCANNER_WRN(txt)  scanner_warning(yylloc, yyextra, txt)

using namespace LinuxSampler;

%}

/* use Flex's built-in support for line numbers */
%option yylineno
/* generate a reentrant safe scanner */
%option reentrant
/* avoid symbol collision with other (i.e. future) scanner symbols */
%option prefix="InstrScript_"
/* bison-bridge adds an argument yylval to yylex, and bison-locations adds an
   argument code yylloc for location tracking. */
%option bison-bridge
%option bison-locations
/* yywrap() would be called at EOF, we don't need it */
%option noyywrap
/* enable functions yy_push_state(), yy_pop_state(), yy_top_state() */
%option stack

/* inclusive scanner conditions */
%s PREPROC_BODY_USE
/* exclusive scanner conditions */
%x PREPROC_SET_COND PREPROC_RESET_COND PREPROC_IF PREPROC_IF_NOT PREPROC_BODY_EAT PREPROC_PRE_BODY_USE PREPROC_PRE_BODY_EAT

DIGIT    [0-9]
ID       [a-zA-Z0-9_]*

%%

\"[^"]*\" {
    yylval->sValue = strdup(yytext + 1);
    yylval->sValue[strlen(yylval->sValue) - 1] = '\0';
    return STRING;
}

{DIGIT}+ {
    yylval->iValue = atoi(yytext);
    return INTEGER;
}

{DIGIT}+"."{DIGIT}* {
    printf("A float: %s (%g)\n", yytext, atof(yytext));
}


 /* Preprocessor statement:  SET_CONDITION(name) */

<*>"SET_CONDITION"[ \t]*"(" {
    //printf("SET_CONDITION\n");
    yy_push_state(PREPROC_SET_COND, yyscanner);
}
<PREPROC_SET_COND>{ID} {
    //printf("preproc set condition '%s'\n", yytext);
    bool success = yyextra->setPreprocessorCondition(yytext);
    if (!success) {
        SCANNER_WRN((String("Preprocessor: Condition '") +
                    yytext + "' is already set.").c_str());
    }
}
<PREPROC_SET_COND>[ \t]*")" {
    //printf("End of PREPROC_SET_COND\n");
    yy_pop_state(yyscanner);
}


 /* Preprocessor statement:  RESET_CONDITION(name) */

<*>"RESET_CONDITION"[ \t]*"(" {
    //printf("RESET_CONDITION\n");
    yy_push_state(PREPROC_RESET_COND, yyscanner);
}
<PREPROC_RESET_COND>{ID} {
    //printf("preproc reset condition '%s'\n", yytext);
    bool success = yyextra->resetPreprocessorCondition(yytext);
    if (!success) {
        SCANNER_ERR((String("Preprocessor: could not reset condition '") +
                    yytext + "' (either not set or a built-in condition).").c_str());
    }
}
<PREPROC_RESET_COND>[ \t]*")" {
    //printf("End of RESET_CONDITION\n");
    yy_pop_state(yyscanner);
}


 /* Preprocessor conditional statements:

    USE_CODE_IF(name)
      ...
    END_USE_CODE

    and:

    USE_CODE_IF_NOT(name)
      ...
    END_USE_CODE    
 */

<*>"USE_CODE_IF"[ \t]*"(" {
    //printf("USE_CODE_IF\n");
    yy_push_state(PREPROC_IF, yyscanner);
}
<*>"USE_CODE_IF_NOT"[ \t]*"(" {
    //printf("USE_CODE_IF_NOT\n");
    yy_push_state(PREPROC_IF_NOT, yyscanner);
}
<PREPROC_IF>{ID} {
    //printf("preproc use code if '%s'\n", yytext);
    yy_pop_state(yyscanner);
    if (yyextra->isPreprocessorConditionSet(yytext))
        yy_push_state(PREPROC_PRE_BODY_USE, yyscanner);
    else
        yy_push_state(PREPROC_PRE_BODY_EAT, yyscanner);
}
<PREPROC_IF_NOT>{ID} {
    //printf("preproc use code if not '%s'\n", yytext);
    yy_pop_state(yyscanner);
    if (!yyextra->isPreprocessorConditionSet(yytext))
        yy_push_state(PREPROC_PRE_BODY_USE, yyscanner);
    else
        yy_push_state(PREPROC_PRE_BODY_EAT, yyscanner);
}
<PREPROC_PRE_BODY_USE>[ \t]*")" {
    yy_pop_state(yyscanner);
    yy_push_state(PREPROC_BODY_USE, yyscanner);
}
<PREPROC_PRE_BODY_EAT>[ \t]*")" {
    //printf("PREPROCESSOR EAT : \n");
    yy_pop_state(yyscanner);
    yy_push_state(PREPROC_BODY_EAT, yyscanner);
}
<*>.*"END_USE_CODE" {
    //printf("-->END_USE_CODE\n");
    yy_pop_state(yyscanner);
}
<PREPROC_BODY_EAT>[ \t\r\n]* /* eat up code block filtered out by preprocessor */
<PREPROC_BODY_EAT>.* /* eat up code block filtered out by preprocessor */


 /* Language keywords */

"on" return ON;
"end" return END;
"note" return NOTE;
"init" return INIT;
"declare" return DECLARE;
"while" return WHILE;
"if" return IF;
"or" return OR;
"release" return RELEASE;
"and" return AND;
"not" return NOT;
"else" return ELSE;
"controller" return CONTROLLER;
"case" return CASE;
"select" return SELECT;
"to" return TO;
"<=" return LE;
">=" return GE;
"const" return CONST_; // note: "CONST" is already defined for C/C++ compilers on Windows by default
"polyphonic" return POLYPHONIC;
"mod" return MOD;

on|end|note|init|declare|if|then|begin|end|procedure|function {
    printf("A keyword: %s\n", yytext);
}

[&,()[\]<>=*+#/-] { return *yytext; }

("$"|"@"){ID} {
    yylval->sValue = strdup(yytext);
    return VARIABLE;
}

"%"{ID} {
    yylval->sValue = strdup(yytext);
    return VARIABLE;
}

{ID} {
    yylval->sValue = strdup(yytext);
    return IDENTIFIER;
}

":=" return ASSIGNMENT;

\n+ {
    //printf("lex: new line %d\n", yylineno, yytext);
    //return LF;
}

"{"[^}]*"}" /* eat up comments */

[ \t\r]+ /* eat up whitespace */

"..." /* eat up */

. printf( "Unrecognized character: %s\n", yytext );

%%

namespace LinuxSampler {

void ParserContext::createScanner(std::istream* is) {
   if (scanner) destroyScanner();
   this->is = is;
   yylex_init(&scanner);
   yyset_extra(this, scanner);
}

void ParserContext::destroyScanner() {
   if (!scanner) return;
   yylex_destroy(scanner);
   scanner = NULL;
}

} // namespace LinuxSampler
1	schoenebeck	2581	/*
2			* Copyright (c) 2014 Christian Schoenebeck and Andreas Persson
3			*
4			* http://www.linuxsampler.org
5			*
6			* This file is part of LinuxSampler and released under the same terms.
7			* See README file for details.
8			*/
9
10			/* Token scanner for instrument script language. */
11
12			/* FIXME: line numbers (i.e. on error/warning messages) are incorrect, because
13			the current grammar does not process new line characters. \n is currently
14			filtered out in this lexer. Due to this, the parser will sometimes read
15			several lines before it matches the next complete grammar rule, causing the
16			incorrect lines in the error/warning messages. */
17
18			%{
19
20			#include "parser_shared.h"
21			#include <math.h>
22			// reentrant scanner data context
23			#define YY_EXTRA_TYPE ParserContext*
24			// set line number each time a token is recognized
25			#define YY_USER_ACTION \
26			{ \
27			yylloc->first_line = yylineno; \
28			yylloc->last_line = yylineno; \
29			/* first_column = TODO */ \
30			/* last_column = TODO */ \
31			/printf("lex: line '%s'\n", yytext);/ \
32			}
33			// custom (f)lex input for reading from std::istream object
34			#define YY_INPUT(buf,result,max_size) \
35			{ \
36			char c = yyextra->is->get(); \
37			if (yyextra->is->eof()) \
38			result = YY_NULL; \
39			else { \
40			buf[0] = c; \
41			result = 1; \
42			} \
43			}
44
45			static void scanner_error(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* err) {
46			context->addErr(locp->first_line, err);
47			}
48
49			static void scanner_warning(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* txt) {
50			context->addWrn(locp->first_line, txt);
51			}
52
53			#define SCANNER_ERR(txt) scanner_error(yylloc, yyextra, txt)
54			#define SCANNER_WRN(txt) scanner_warning(yylloc, yyextra, txt)
55
56			using namespace LinuxSampler;
57
58			%}
59
60			/* use Flex's built-in support for line numbers */
61			%option yylineno
62			/* generate a reentrant safe scanner */
63			%option reentrant
64			/* avoid symbol collision with other (i.e. future) scanner symbols */
65			%option prefix="InstrScript_"
66			/* bison-bridge adds an argument yylval to yylex, and bison-locations adds an
67			argument code yylloc for location tracking. */
68			%option bison-bridge
69			%option bison-locations
70			/* yywrap() would be called at EOF, we don't need it */
71			%option noyywrap
72			/* enable functions yy_push_state(), yy_pop_state(), yy_top_state() */
73			%option stack
74
75			/* inclusive scanner conditions */
76			%s PREPROC_BODY_USE
77			/* exclusive scanner conditions */
78			%x PREPROC_SET_COND PREPROC_RESET_COND PREPROC_IF PREPROC_IF_NOT PREPROC_BODY_EAT PREPROC_PRE_BODY_USE PREPROC_PRE_BODY_EAT
79
80			DIGIT [0-9]
81			ID [a-zA-Z0-9_]*
82
83			%%
84
85			\"[^"]*\" {
86			yylval->sValue = strdup(yytext + 1);
87			yylval->sValue[strlen(yylval->sValue) - 1] = '\0';
88			return STRING;
89			}
90
91			{DIGIT}+ {
92			yylval->iValue = atoi(yytext);
93			return INTEGER;
94			}
95
96			{DIGIT}+"."{DIGIT}* {
97			printf("A float: %s (%g)\n", yytext, atof(yytext));
98			}
99
100
101			/* Preprocessor statement: SET_CONDITION(name) */
102
103			<>"SET_CONDITION"[ \t]"(" {
104	schoenebeck	2588	//printf("SET_CONDITION\n");
105	schoenebeck	2581	yy_push_state(PREPROC_SET_COND, yyscanner);
106			}
107			<PREPROC_SET_COND>{ID} {
108	schoenebeck	2588	//printf("preproc set condition '%s'\n", yytext);
109	schoenebeck	2581	bool success = yyextra->setPreprocessorCondition(yytext);
110			if (!success) {
111			SCANNER_WRN((String("Preprocessor: Condition '") +
112			yytext + "' is already set.").c_str());
113			}
114			}
115			<PREPROC_SET_COND>[ \t]*")" {
116	schoenebeck	2588	//printf("End of PREPROC_SET_COND\n");
117	schoenebeck	2581	yy_pop_state(yyscanner);
118			}
119
120
121			/* Preprocessor statement: RESET_CONDITION(name) */
122
123			<>"RESET_CONDITION"[ \t]"(" {
124	schoenebeck	2588	//printf("RESET_CONDITION\n");
125	schoenebeck	2581	yy_push_state(PREPROC_RESET_COND, yyscanner);
126			}
127			<PREPROC_RESET_COND>{ID} {
128	schoenebeck	2588	//printf("preproc reset condition '%s'\n", yytext);
129	schoenebeck	2581	bool success = yyextra->resetPreprocessorCondition(yytext);
130			if (!success) {
131			SCANNER_ERR((String("Preprocessor: could not reset condition '") +
132			yytext + "' (either not set or a built-in condition).").c_str());
133			}
134			}
135			<PREPROC_RESET_COND>[ \t]*")" {
136	schoenebeck	2588	//printf("End of RESET_CONDITION\n");
137	schoenebeck	2581	yy_pop_state(yyscanner);
138			}
139
140
141			/* Preprocessor conditional statements:
142
143			USE_CODE_IF(name)
144			...
145			END_USE_CODE
146
147			and:
148
149			USE_CODE_IF_NOT(name)
150			...
151			END_USE_CODE
152			*/
153
154			<>"USE_CODE_IF"[ \t]"(" {
155	schoenebeck	2588	//printf("USE_CODE_IF\n");
156	schoenebeck	2581	yy_push_state(PREPROC_IF, yyscanner);
157			}
158			<>"USE_CODE_IF_NOT"[ \t]"(" {
159	schoenebeck	2588	//printf("USE_CODE_IF_NOT\n");
160	schoenebeck	2581	yy_push_state(PREPROC_IF_NOT, yyscanner);
161			}
162			<PREPROC_IF>{ID} {
163	schoenebeck	2588	//printf("preproc use code if '%s'\n", yytext);
164	schoenebeck	2581	yy_pop_state(yyscanner);
165			if (yyextra->isPreprocessorConditionSet(yytext))
166			yy_push_state(PREPROC_PRE_BODY_USE, yyscanner);
167			else
168			yy_push_state(PREPROC_PRE_BODY_EAT, yyscanner);
169			}
170			<PREPROC_IF_NOT>{ID} {
171	schoenebeck	2588	//printf("preproc use code if not '%s'\n", yytext);
172	schoenebeck	2581	yy_pop_state(yyscanner);
173			if (!yyextra->isPreprocessorConditionSet(yytext))
174			yy_push_state(PREPROC_PRE_BODY_USE, yyscanner);
175			else
176			yy_push_state(PREPROC_PRE_BODY_EAT, yyscanner);
177			}
178			<PREPROC_PRE_BODY_USE>[ \t]*")" {
179			yy_pop_state(yyscanner);
180			yy_push_state(PREPROC_BODY_USE, yyscanner);
181			}
182			<PREPROC_PRE_BODY_EAT>[ \t]*")" {
183	schoenebeck	2588	//printf("PREPROCESSOR EAT : \n");
184	schoenebeck	2581	yy_pop_state(yyscanner);
185			yy_push_state(PREPROC_BODY_EAT, yyscanner);
186			}
187			<>."END_USE_CODE" {
188	schoenebeck	2588	//printf("-->END_USE_CODE\n");
189	schoenebeck	2581	yy_pop_state(yyscanner);
190			}
191	schoenebeck	2588	<PREPROC_BODY_EAT>[ \t\r\n]* /* eat up code block filtered out by preprocessor */
192			<PREPROC_BODY_EAT>.* /* eat up code block filtered out by preprocessor */
193	schoenebeck	2581
194
195			/* Language keywords */
196
197			"on" return ON;
198			"end" return END;
199			"note" return NOTE;
200			"init" return INIT;
201			"declare" return DECLARE;
202			"while" return WHILE;
203			"if" return IF;
204			"or" return OR;
205			"release" return RELEASE;
206			"and" return AND;
207			"not" return NOT;
208			"else" return ELSE;
209			"controller" return CONTROLLER;
210			"case" return CASE;
211			"select" return SELECT;
212			"to" return TO;
213			"<=" return LE;
214			">=" return GE;
215	schoenebeck	2585	"const" return CONST_; // note: "CONST" is already defined for C/C++ compilers on Windows by default
216	schoenebeck	2581	"polyphonic" return POLYPHONIC;
217			"mod" return MOD;
218
219			on\|end\|note\|init\|declare\|if\|then\|begin\|end\|procedure\|function {
220			printf("A keyword: %s\n", yytext);
221			}
222
223			[&,()[\]<>=+#/-] { return yytext; }
224
225			("$"\|"@"){ID} {
226			yylval->sValue = strdup(yytext);
227			return VARIABLE;
228			}
229
230			"%"{ID} {
231			yylval->sValue = strdup(yytext);
232			return VARIABLE;
233			}
234
235			{ID} {
236			yylval->sValue = strdup(yytext);
237			return IDENTIFIER;
238			}
239
240			":=" return ASSIGNMENT;
241
242			\n+ {
243			//printf("lex: new line %d\n", yylineno, yytext);
244			//return LF;
245			}
246
247			"{"[^}]"}" / eat up comments */
248
249			[ \t\r]+ /* eat up whitespace */
250
251			"..." /* eat up */
252
253			. printf( "Unrecognized character: %s\n", yytext );
254
255			%%
256
257			namespace LinuxSampler {
258
259			void ParserContext::createScanner(std::istream* is) {
260			if (scanner) destroyScanner();
261			this->is = is;
262			yylex_init(&scanner);
263			yyset_extra(this, scanner);
264			}
265
266			void ParserContext::destroyScanner() {
267			if (!scanner) return;
268			yylex_destroy(scanner);
269			scanner = NULL;
270			}
271
272			} // namespace LinuxSampler