src/scriptvm/scanner.l

/*
 * Copyright (c) 2014 Christian Schoenebeck and Andreas Persson
 *
 * http://www.linuxsampler.org
 *
 * This file is part of LinuxSampler and released under the same terms.
 * See README file for details.
 */

/* Token scanner for instrument script language. */

/* FIXME: line numbers (i.e. on error/warning messages) are incorrect, because
   the current grammar does not process new line characters. \n is currently
   filtered out in this lexer. Due to this, the parser will sometimes read
   several lines before it matches the next complete grammar rule, causing the
   incorrect lines in the error/warning messages. */

%{

#include "parser_shared.h"
#include <math.h>
// reentrant scanner data context
#define YY_EXTRA_TYPE ParserContext*
// set line number each time a token is recognized
#define YY_USER_ACTION                  \
{                                       \
    yylloc->first_line = yylineno;      \
    yylloc->last_line  = yylineno;      \
    /* first_column = TODO */           \
    /* last_column  = TODO */           \
    /*printf("lex: line '%s'\n", yytext);*/  \
}
// custom (f)lex input for reading from std::istream object
#define YY_INPUT(buf,result,max_size)   \
{                                       \
    char c = yyextra->is->get();        \
    if (yyextra->is->eof())             \
        result = YY_NULL;               \
    else {                              \
        buf[0] = c;                     \
        result = 1;                     \
    }                                   \
}

static void scanner_error(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* err) {
    context->addErr(locp->first_line, err);
}

static void scanner_warning(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* txt) {
    context->addWrn(locp->first_line, txt);
}

#define SCANNER_ERR(txt)  scanner_error(yylloc, yyextra, txt)
#define SCANNER_WRN(txt)  scanner_warning(yylloc, yyextra, txt)

using namespace LinuxSampler;

%}

/* use Flex's built-in support for line numbers */
%option yylineno
/* generate a reentrant safe scanner */
%option reentrant
/* avoid symbol collision with other (i.e. future) scanner symbols */
%option prefix="InstrScript_"
/* bison-bridge adds an argument yylval to yylex, and bison-locations adds an
   argument code yylloc for location tracking. */
%option bison-bridge
%option bison-locations
/* yywrap() would be called at EOF, we don't need it */
%option noyywrap
/* enable functions yy_push_state(), yy_pop_state(), yy_top_state() */
%option stack

/* inclusive scanner conditions */
%s PREPROC_BODY_USE
/* exclusive scanner conditions */
%x PREPROC_SET_COND PREPROC_RESET_COND PREPROC_IF PREPROC_IF_NOT PREPROC_BODY_EAT PREPROC_PRE_BODY_USE PREPROC_PRE_BODY_EAT

DIGIT    [0-9]
ID       [a-zA-Z0-9_]*

%%

\"[^"]*\" {
    yylval->sValue = strdup(yytext + 1);
    yylval->sValue[strlen(yylval->sValue) - 1] = '\0';
    return STRING;
}

{DIGIT}+ {
    yylval->iValue = atoi(yytext);
    return INTEGER;
}

{DIGIT}+"."{DIGIT}* {
    printf("A float: %s (%g)\n", yytext, atof(yytext));
}


 /* Preprocessor statement:  SET_CONDITION(name) */

<*>"SET_CONDITION"[ \t]*"(" {
    printf("SET_CONDITION\n");
    yy_push_state(PREPROC_SET_COND, yyscanner);
}
<PREPROC_SET_COND>{ID} {
    printf("preproc set condition '%s'\n", yytext);
    bool success = yyextra->setPreprocessorCondition(yytext);
    if (!success) {
        SCANNER_WRN((String("Preprocessor: Condition '") +
                    yytext + "' is already set.").c_str());
    }
}
<PREPROC_SET_COND>[ \t]*")" {
    printf("End of PREPROC_SET_COND\n");
    yy_pop_state(yyscanner);
}


 /* Preprocessor statement:  RESET_CONDITION(name) */

<*>"RESET_CONDITION"[ \t]*"(" {
    printf("RESET_CONDITION\n");
    yy_push_state(PREPROC_RESET_COND, yyscanner);
}
<PREPROC_RESET_COND>{ID} {
    printf("preproc reset condition '%s'\n", yytext);
    bool success = yyextra->resetPreprocessorCondition(yytext);
    if (!success) {
        SCANNER_ERR((String("Preprocessor: could not reset condition '") +
                    yytext + "' (either not set or a built-in condition).").c_str());
    }
}
<PREPROC_RESET_COND>[ \t]*")" {
    printf("End of RESET_CONDITION\n");
    yy_pop_state(yyscanner);
}


 /* Preprocessor conditional statements:

    USE_CODE_IF(name)
      ...
    END_USE_CODE

    and:

    USE_CODE_IF_NOT(name)
      ...
    END_USE_CODE    
 */

<*>"USE_CODE_IF"[ \t]*"(" {
    printf("USE_CODE_IF\n");
    yy_push_state(PREPROC_IF, yyscanner);
}
<*>"USE_CODE_IF_NOT"[ \t]*"(" {
    printf("USE_CODE_IF_NOT\n");
    yy_push_state(PREPROC_IF_NOT, yyscanner);
}
<PREPROC_IF>{ID} {
    printf("preproc use code if '%s'\n", yytext);
    yy_pop_state(yyscanner);
    if (yyextra->isPreprocessorConditionSet(yytext))
        yy_push_state(PREPROC_PRE_BODY_USE, yyscanner);
    else
        yy_push_state(PREPROC_PRE_BODY_EAT, yyscanner);
}
<PREPROC_IF_NOT>{ID} {
    printf("preproc use code if not '%s'\n", yytext);
    yy_pop_state(yyscanner);
    if (!yyextra->isPreprocessorConditionSet(yytext))
        yy_push_state(PREPROC_PRE_BODY_USE, yyscanner);
    else
        yy_push_state(PREPROC_PRE_BODY_EAT, yyscanner);
}
<PREPROC_PRE_BODY_USE>[ \t]*")" {
    yy_pop_state(yyscanner);
    yy_push_state(PREPROC_BODY_USE, yyscanner);
}
<PREPROC_PRE_BODY_EAT>[ \t]*")" {
    yy_pop_state(yyscanner);
    yy_push_state(PREPROC_BODY_EAT, yyscanner);
}
<*>.*"END_USE_CODE" {
    printf("END_USE_CODE\n");
    yy_pop_state(yyscanner);
}


 /* Language keywords */

"on" return ON;
"end" return END;
"note" return NOTE;
"init" return INIT;
"declare" return DECLARE;
"while" return WHILE;
"if" return IF;
"or" return OR;
"release" return RELEASE;
"and" return AND;
"not" return NOT;
"else" return ELSE;
"controller" return CONTROLLER;
"case" return CASE;
"select" return SELECT;
"to" return TO;
"<=" return LE;
">=" return GE;
"const" return CONST;
"polyphonic" return POLYPHONIC;
"mod" return MOD;

on|end|note|init|declare|if|then|begin|end|procedure|function {
    printf("A keyword: %s\n", yytext);
}

[&,()[\]<>=*+#/-] { return *yytext; }

("$"|"@"){ID} {
    yylval->sValue = strdup(yytext);
    return VARIABLE;
}

"%"{ID} {
    yylval->sValue = strdup(yytext);
    return VARIABLE;
}

{ID} {
    yylval->sValue = strdup(yytext);
    return IDENTIFIER;
}

":=" return ASSIGNMENT;

\n+ {
    //printf("lex: new line %d\n", yylineno, yytext);
    //return LF;
}

"{"[^}]*"}" /* eat up comments */

[ \t\r]+ /* eat up whitespace */

"..." /* eat up */

. printf( "Unrecognized character: %s\n", yytext );

%%

namespace LinuxSampler {

void ParserContext::createScanner(std::istream* is) {
   if (scanner) destroyScanner();
   this->is = is;
   yylex_init(&scanner);
   yyset_extra(this, scanner);
}

void ParserContext::destroyScanner() {
   if (!scanner) return;
   yylex_destroy(scanner);
   scanner = NULL;
}

} // namespace LinuxSampler
1	schoenebeck	2581	/*
2			* Copyright (c) 2014 Christian Schoenebeck and Andreas Persson
3			*
4			* http://www.linuxsampler.org
5			*
6			* This file is part of LinuxSampler and released under the same terms.
7			* See README file for details.
8			*/
9
10			/* Token scanner for instrument script language. */
11
12			/* FIXME: line numbers (i.e. on error/warning messages) are incorrect, because
13			the current grammar does not process new line characters. \n is currently
14			filtered out in this lexer. Due to this, the parser will sometimes read
15			several lines before it matches the next complete grammar rule, causing the
16			incorrect lines in the error/warning messages. */
17
18			%{
19
20			#include "parser_shared.h"
21			#include <math.h>
22			// reentrant scanner data context
23			#define YY_EXTRA_TYPE ParserContext*
24			// set line number each time a token is recognized
25			#define YY_USER_ACTION \
26			{ \
27			yylloc->first_line = yylineno; \
28			yylloc->last_line = yylineno; \
29			/* first_column = TODO */ \
30			/* last_column = TODO */ \
31			/printf("lex: line '%s'\n", yytext);/ \
32			}
33			// custom (f)lex input for reading from std::istream object
34			#define YY_INPUT(buf,result,max_size) \
35			{ \
36			char c = yyextra->is->get(); \
37			if (yyextra->is->eof()) \
38			result = YY_NULL; \
39			else { \
40			buf[0] = c; \
41			result = 1; \
42			} \
43			}
44
45			static void scanner_error(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* err) {
46			context->addErr(locp->first_line, err);
47			}
48
49			static void scanner_warning(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* txt) {
50			context->addWrn(locp->first_line, txt);
51			}
52
53			#define SCANNER_ERR(txt) scanner_error(yylloc, yyextra, txt)
54			#define SCANNER_WRN(txt) scanner_warning(yylloc, yyextra, txt)
55
56			using namespace LinuxSampler;
57
58			%}
59
60			/* use Flex's built-in support for line numbers */
61			%option yylineno
62			/* generate a reentrant safe scanner */
63			%option reentrant
64			/* avoid symbol collision with other (i.e. future) scanner symbols */
65			%option prefix="InstrScript_"
66			/* bison-bridge adds an argument yylval to yylex, and bison-locations adds an
67			argument code yylloc for location tracking. */
68			%option bison-bridge
69			%option bison-locations
70			/* yywrap() would be called at EOF, we don't need it */
71			%option noyywrap
72			/* enable functions yy_push_state(), yy_pop_state(), yy_top_state() */
73			%option stack
74
75			/* inclusive scanner conditions */
76			%s PREPROC_BODY_USE
77			/* exclusive scanner conditions */
78			%x PREPROC_SET_COND PREPROC_RESET_COND PREPROC_IF PREPROC_IF_NOT PREPROC_BODY_EAT PREPROC_PRE_BODY_USE PREPROC_PRE_BODY_EAT
79
80			DIGIT [0-9]
81			ID [a-zA-Z0-9_]*
82
83			%%
84
85			\"[^"]*\" {
86			yylval->sValue = strdup(yytext + 1);
87			yylval->sValue[strlen(yylval->sValue) - 1] = '\0';
88			return STRING;
89			}
90
91			{DIGIT}+ {
92			yylval->iValue = atoi(yytext);
93			return INTEGER;
94			}
95
96			{DIGIT}+"."{DIGIT}* {
97			printf("A float: %s (%g)\n", yytext, atof(yytext));
98			}
99
100
101			/* Preprocessor statement: SET_CONDITION(name) */
102
103			<>"SET_CONDITION"[ \t]"(" {
104			printf("SET_CONDITION\n");
105			yy_push_state(PREPROC_SET_COND, yyscanner);
106			}
107			<PREPROC_SET_COND>{ID} {
108			printf("preproc set condition '%s'\n", yytext);
109			bool success = yyextra->setPreprocessorCondition(yytext);
110			if (!success) {
111			SCANNER_WRN((String("Preprocessor: Condition '") +
112			yytext + "' is already set.").c_str());
113			}
114			}
115			<PREPROC_SET_COND>[ \t]*")" {
116			printf("End of PREPROC_SET_COND\n");
117			yy_pop_state(yyscanner);
118			}
119
120
121			/* Preprocessor statement: RESET_CONDITION(name) */
122
123			<>"RESET_CONDITION"[ \t]"(" {
124			printf("RESET_CONDITION\n");
125			yy_push_state(PREPROC_RESET_COND, yyscanner);
126			}
127			<PREPROC_RESET_COND>{ID} {
128			printf("preproc reset condition '%s'\n", yytext);
129			bool success = yyextra->resetPreprocessorCondition(yytext);
130			if (!success) {
131			SCANNER_ERR((String("Preprocessor: could not reset condition '") +
132			yytext + "' (either not set or a built-in condition).").c_str());
133			}
134			}
135			<PREPROC_RESET_COND>[ \t]*")" {
136			printf("End of RESET_CONDITION\n");
137			yy_pop_state(yyscanner);
138			}
139
140
141			/* Preprocessor conditional statements:
142
143			USE_CODE_IF(name)
144			...
145			END_USE_CODE
146
147			and:
148
149			USE_CODE_IF_NOT(name)
150			...
151			END_USE_CODE
152			*/
153
154			<>"USE_CODE_IF"[ \t]"(" {
155			printf("USE_CODE_IF\n");
156			yy_push_state(PREPROC_IF, yyscanner);
157			}
158			<>"USE_CODE_IF_NOT"[ \t]"(" {
159			printf("USE_CODE_IF_NOT\n");
160			yy_push_state(PREPROC_IF_NOT, yyscanner);
161			}
162			<PREPROC_IF>{ID} {
163			printf("preproc use code if '%s'\n", yytext);
164			yy_pop_state(yyscanner);
165			if (yyextra->isPreprocessorConditionSet(yytext))
166			yy_push_state(PREPROC_PRE_BODY_USE, yyscanner);
167			else
168			yy_push_state(PREPROC_PRE_BODY_EAT, yyscanner);
169			}
170			<PREPROC_IF_NOT>{ID} {
171			printf("preproc use code if not '%s'\n", yytext);
172			yy_pop_state(yyscanner);
173			if (!yyextra->isPreprocessorConditionSet(yytext))
174			yy_push_state(PREPROC_PRE_BODY_USE, yyscanner);
175			else
176			yy_push_state(PREPROC_PRE_BODY_EAT, yyscanner);
177			}
178			<PREPROC_PRE_BODY_USE>[ \t]*")" {
179			yy_pop_state(yyscanner);
180			yy_push_state(PREPROC_BODY_USE, yyscanner);
181			}
182			<PREPROC_PRE_BODY_EAT>[ \t]*")" {
183			yy_pop_state(yyscanner);
184			yy_push_state(PREPROC_BODY_EAT, yyscanner);
185			}
186			<>."END_USE_CODE" {
187			printf("END_USE_CODE\n");
188			yy_pop_state(yyscanner);
189			}
190
191
192			/* Language keywords */
193
194			"on" return ON;
195			"end" return END;
196			"note" return NOTE;
197			"init" return INIT;
198			"declare" return DECLARE;
199			"while" return WHILE;
200			"if" return IF;
201			"or" return OR;
202			"release" return RELEASE;
203			"and" return AND;
204			"not" return NOT;
205			"else" return ELSE;
206			"controller" return CONTROLLER;
207			"case" return CASE;
208			"select" return SELECT;
209			"to" return TO;
210			"<=" return LE;
211			">=" return GE;
212			"const" return CONST;
213			"polyphonic" return POLYPHONIC;
214			"mod" return MOD;
215
216			on\|end\|note\|init\|declare\|if\|then\|begin\|end\|procedure\|function {
217			printf("A keyword: %s\n", yytext);
218			}
219
220			[&,()[\]<>=+#/-] { return yytext; }
221
222			("$"\|"@"){ID} {
223			yylval->sValue = strdup(yytext);
224			return VARIABLE;
225			}
226
227			"%"{ID} {
228			yylval->sValue = strdup(yytext);
229			return VARIABLE;
230			}
231
232			{ID} {
233			yylval->sValue = strdup(yytext);
234			return IDENTIFIER;
235			}
236
237			":=" return ASSIGNMENT;
238
239			\n+ {
240			//printf("lex: new line %d\n", yylineno, yytext);
241			//return LF;
242			}
243
244			"{"[^}]"}" / eat up comments */
245
246			[ \t\r]+ /* eat up whitespace */
247
248			"..." /* eat up */
249
250			. printf( "Unrecognized character: %s\n", yytext );
251
252			%%
253
254			namespace LinuxSampler {
255
256			void ParserContext::createScanner(std::istream* is) {
257			if (scanner) destroyScanner();
258			this->is = is;
259			yylex_init(&scanner);
260			yyset_extra(this, scanner);
261			}
262
263			void ParserContext::destroyScanner() {
264			if (!scanner) return;
265			yylex_destroy(scanner);
266			scanner = NULL;
267			}
268
269			} // namespace LinuxSampler