--- linuxsampler/trunk/src/scriptvm/scanner.l	2014/06/01 14:44:38	2588
+++ linuxsampler/trunk/src/scriptvm/scanner.l	2017/05/31 21:07:44	3260
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014 Christian Schoenebeck and Andreas Persson
+ * Copyright (c) 2014-2017 Christian Schoenebeck and Andreas Persson
  *
  * http://www.linuxsampler.org
  *
@@ -7,13 +7,7 @@
  * See README file for details.
  */
 
-/* Token scanner for instrument script language. */
-
-/* FIXME: line numbers (i.e. on error/warning messages) are incorrect, because
-   the current grammar does not process new line characters. \n is currently
-   filtered out in this lexer. Due to this, the parser will sometimes read
-   several lines before it matches the next complete grammar rule, causing the
-   incorrect lines in the error/warning messages. */
+/* Token scanner for NKSP real-time instrument script language. */
 
 %{
 
@@ -26,8 +20,9 @@
 {                                       \
     yylloc->first_line = yylineno;      \
     yylloc->last_line  = yylineno;      \
-    /* first_column = TODO */           \
-    /* last_column  = TODO */           \
+    yylloc->first_column = yycolumn;    \
+    yycolumn += yyleng;                 \
+    yylloc->last_column = yycolumn - 1; \
     /*printf("lex: line '%s'\n", yytext);*/  \
 }
 // custom (f)lex input for reading from std::istream object
@@ -43,22 +38,50 @@
 }
 
 static void scanner_error(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* err) {
-    context->addErr(locp->first_line, err);
+    context->addErr(locp->first_line, locp->last_line, locp->first_column, locp->last_column, err);
 }
 
 static void scanner_warning(YYLTYPE* locp, LinuxSampler::ParserContext* context, const char* txt) {
-    context->addWrn(locp->first_line, txt);
+    context->addWrn(locp->first_line, locp->last_line, locp->first_column, locp->last_column, txt);
 }
 
 #define SCANNER_ERR(txt)  scanner_error(yylloc, yyextra, txt)
 #define SCANNER_WRN(txt)  scanner_warning(yylloc, yyextra, txt)
 
+// shut up warning that 'register' keyword is deprecated as of C++11
+#if defined(__cplusplus) && __cplusplus >= 201103L
+# define register
+#endif
+
 using namespace LinuxSampler;
 
+static int countNewLineChars(const char* txt) {
+    int n = 0;
+    for (int i = 0; txt[i]; ++i)
+        if (txt[i] == '\n') ++n;
+    return n;
+}
+
+static int countCharsPastLastNewLine(const char* txt) {
+    const int n = (int)strlen(txt);
+    for (int i = n - 1; i >= 0; --i)
+        if (txt[i] == '\n')
+            return n - i - 1;
+    return n;
+}
+
+#define processLocation() { \
+    const int nl = countNewLineChars(yytext); \
+    yylineno += nl; \
+    if (nl) yycolumn = countCharsPastLastNewLine(yytext); \
+}
+
 %}
 
-/* use Flex's built-in support for line numbers */
-%option yylineno
+/* use Flex's built-in support for line numbers
+   (disabled, because it seems to be unreliable, so we are using our own
+   tracking code in the respective scanner rules below) */
+/*%option yylineno*/
 /* generate a reentrant safe scanner */
 %option reentrant
 /* avoid symbol collision with other (i.e. future) scanner symbols */
@@ -78,7 +101,7 @@
 %x PREPROC_SET_COND PREPROC_RESET_COND PREPROC_IF PREPROC_IF_NOT PREPROC_BODY_EAT PREPROC_PRE_BODY_USE PREPROC_PRE_BODY_EAT
 
 DIGIT    [0-9]
-ID       [a-zA-Z0-9_]*
+ID       [a-zA-Z0-9_]+
 
 %%
 
@@ -93,9 +116,10 @@
     return INTEGER;
 }
 
-{DIGIT}+"."{DIGIT}* {
+ /* there is currently no support for floating point numbers in NKSP yet */
+ /*{DIGIT}+"."{DIGIT}* {
     printf("A float: %s (%g)\n", yytext, atof(yytext));
-}
+ }*/
 
 
  /* Preprocessor statement:  SET_CONDITION(name) */
@@ -188,8 +212,12 @@
     //printf("-->END_USE_CODE\n");
     yy_pop_state(yyscanner);
 }
-<PREPROC_BODY_EAT>[ \t\r\n]* /* eat up code block filtered out by preprocessor */
-<PREPROC_BODY_EAT>.* /* eat up code block filtered out by preprocessor */
+<PREPROC_BODY_EAT>[ \t\r\n]* { /* eat up code block filtered out by preprocessor */
+    processLocation();
+}
+<PREPROC_BODY_EAT>.* { /* eat up code block filtered out by preprocessor */
+    processLocation();
+}
 
 
  /* Language keywords */
@@ -201,9 +229,12 @@
 "declare" return DECLARE;
 "while" return WHILE;
 "if" return IF;
+".or." return BITWISE_OR;
 "or" return OR;
 "release" return RELEASE;
+".and." return BITWISE_AND;
 "and" return AND;
+".not." return BITWISE_NOT;
 "not" return NOT;
 "else" return ELSE;
 "controller" return CONTROLLER;
@@ -215,19 +246,15 @@
 "const" return CONST_; // note: "CONST" is already defined for C/C++ compilers on Windows by default
 "polyphonic" return POLYPHONIC;
 "mod" return MOD;
+"function" return FUNCTION;
+"call" return CALL;
+"synchronized" return SYNCHRONIZED;
 
-on|end|note|init|declare|if|then|begin|end|procedure|function {
-    printf("A keyword: %s\n", yytext);
+[&,()[\]<>=*+#/-] {
+    return *yytext;
 }
 
-[&,()[\]<>=*+#/-] { return *yytext; }
-
-("$"|"@"){ID} {
-    yylval->sValue = strdup(yytext);
-    return VARIABLE;
-}
-
-"%"{ID} {
+("$"|"@"|"%"){ID} {
     yylval->sValue = strdup(yytext);
     return VARIABLE;
 }
@@ -240,17 +267,22 @@
 ":=" return ASSIGNMENT;
 
 \n+ {
+     yylineno += countNewLineChars(yytext);
+     yycolumn = 0;
     //printf("lex: new line %d\n", yylineno, yytext);
     //return LF;
 }
 
-"{"[^}]*"}" /* eat up comments */
+"{"[^}]*"}" { /* eat up comments */
+    processLocation();
+}
 
 [ \t\r]+ /* eat up whitespace */
 
-"..." /* eat up */
-
-. printf( "Unrecognized character: %s\n", yytext );
+. {
+    printf( "Unrecognized character: '%s' (line %d, column %d)\n", yytext, yylineno, yycolumn);
+    return UNKNOWN_CHAR;
+}
 
 %%