IndexOutOfRangeException in private void SetNewPosition() sourceStream.cs

Dec 9, 2013 at 8:34 PM
im using an Actipro textbox with Irony integration sample setup. I have a simple SQL type grrammer. But when parsing i seem to be randomly getting an IndexOutofRange exception from this function in SourceStream.cs It seems to be when certain text is in the text box, or if im deleting text form the text box. Actually this might be part of actipros code and not even Irony related. Anyway great project its very useful. Hoping maybe someone has seen this issue before.
thanks.
  private void SetNewPosition(int newPosition) {
      if (newPosition < Position)
        throw new Exception(Resources.ErrCannotMoveBackInSource); 
      int p = Position; 
      int col = Location.Column;
      int line = Location.Line; 
      while(p <  newPosition) {
        var curr = _chars[p];     <----------------ERROR occurring here
        switch (curr) {
          case '\n': line++; col = 0; break;
          case '\r': break; 
          case '\t': col = (col / _tabWidth + 1) * _tabWidth;     break;
          default: col++; break; 
        } //switch
        p++;
      }
      Location = new SourceLocation(p, line, col); 
    }
Dec 10, 2013 at 8:12 PM
Edited Dec 10, 2013 at 8:13 PM
I actually have a feeling its a problem with my grammar:
(I am very new to this so there are potentially many issues)

the following string produces the error:
[a].[b] = "
  // Terminals (Lexing)
            NumberLiteral number = new NumberLiteral("number");
            StringLiteral STRING = new StringLiteral("STRING", "\"", StringOptions.IsTemplate);

            //Let's allow big integers (with unlimited number of digits):
            number.DefaultIntTypes = new TypeCode[] { TypeCode.Int32, TypeCode.Int64, NumberLiteral.TypeCodeBigInt };
            IdentifierTerminal Name = new IdentifierTerminal("Name");
            //var Name = TerminalFactory.CreateSqlExtIdentifier(this, "id_simple");
            CommentTerminal comment = new CommentTerminal("comment", "//", "\n", "\r");
            //comment must be added to NonGrammarTerminals list; it is not used directly in grammar rules,
            // so we add it to this list to let Scanner know that it is also a valid terminal. 
            NonGrammarTerminals.Add(comment);
            comment = new CommentTerminal("multilineComment", "/*", "*/");
            NonGrammarTerminals.Add(comment);

            ConstantTerminal CONSTANT = new ConstantTerminal("CONSTANT");
            CONSTANT.Add("NULL", null);


            // Non-Terminals (Parsing)
            NonTerminal query = new NonTerminal("Query");
            NonTerminal tableExpression = new NonTerminal("TableExpression");
            NonTerminal tableExpressions = new NonTerminal("TableExpressions");
            NonTerminal table = new NonTerminal("Table");
            NonTerminal column = new NonTerminal("Column");
            NonTerminal tableOperator = new NonTerminal("TableOperator");
            NonTerminal value = new NonTerminal("Value");
            NonTerminal logicOp = new NonTerminal("LogicOp");
            NonTerminal parameter = new NonTerminal("Parameter");
            NonTerminal list = new NonTerminal("List");
            NonTerminal enclosure = new NonTerminal("Enclosure");
            NonTerminal closure = new NonTerminal("Closure");
            NonTerminal logicExpression = new NonTerminal("logicExpression");
            NonTerminal queryExpression = new NonTerminal("queryExpression");
            NonTerminal betweenStmt = new NonTerminal("BetweenStmt");
            NonTerminal expList = new NonTerminal("ExpList");

            //keywords
            KeyTerm AND = ToTerm("AND");
            KeyTerm OR = ToTerm("OR");
            KeyTerm IN = ToTerm("IN");
            KeyTerm BETWEEN = ToTerm("BETWEEN");
            KeyTerm LIKE = ToTerm("LIKE");
            KeyTerm NOT = ToTerm("NOT");
            KeyTerm dot = ToTerm(".", "dot");
            KeyTerm comma = ToTerm(",", "comma");
            KeyTerm LeftSquareBrace = ToTerm("[", "LeftSquareBrace");
            KeyTerm RightSquareBrace = ToTerm("]", "RightSquareBrace");
            KeyTerm LeftCurlyBrace = ToTerm("{", "LeftSCurlyBrace");
            KeyTerm RightCurlyBrace = ToTerm("}", "RightCurlyBrace");
            KeyTerm LeftQuote = ToTerm("\"", "LeftQuote");
            KeyTerm RightQuote = ToTerm("\"", "RightQuote");


            MarkPunctuation(",", "(", ")", "[", "]", ".", "\"", "{", "}");

            logicExpression.Rule = tableExpression + logicOp + tableExpression | logicOp + tableExpression + logicOp | tableExpression + logicOp | logicOp + tableExpression | tableExpression;
            queryExpression.Rule = logicExpression | "(" + logicExpression + ")";
            tableExpression.Rule = table + dot + column + tableOperator + value;
            tableExpression.ErrorRule = SyntaxError + ";";
            betweenStmt.Rule = BETWEEN + value + "AND";
            tableOperator.Rule = ToTerm("=") | ">" | "<" | "<>" | ">=" | "<=" | "LIKE" | "IN" | "NOT LIKE" | "IS" | "IS NOT" | betweenStmt;
            value.Rule = number | parameter | STRING | CONSTANT | expList;
            enclosure.Rule = ToTerm("(") | Empty;
            closure.Rule = ToTerm(")") | Empty;
            parameter.Rule = LeftQuote + "{" + Name + "}" + "\"" | "{" + Name + "}" | "#" + "{" + Name + "}" + "#";
            logicOp.Rule = AND | OR;
            expList.Rule = "(" + list + ")";
            list.Rule = MakePlusRule(list, comma, value);
            table.Rule = LeftSquareBrace + Name + RightSquareBrace;
            table.ErrorRule = SyntaxError + ";";
            column.Rule = LeftSquareBrace + Name + RightSquareBrace;
            column.ErrorRule = SyntaxError + ";";
            query.Rule = MakePlusRule(query, queryExpression);

            Root = query;
the following string produces the error:

[a].[b] = "
Coordinator
Dec 11, 2013 at 11:46 PM
did you run this thru Grammar Explorer?! it shows 1 error and 3 shift/reduce conflicts
First clean up this, then try to parse
Coordinator
Dec 12, 2013 at 12:14 AM
I see some issues:
remove IsTemplate parameter from string terminal constructor
Declare precedence using RegisterOperators, smth like:
        RegisterOperators(90, AND);
        RegisterOperators(80, OR);
        RegisterOperators(70, "=", ">", "<", "<>", ">=", "<=", "IN", "LIKE", "NOT LIKE", "IS", "IS NOT", "BETWEEN");
And remaining conflict I think comes from this really strange definition:
        logicExpression.Rule = tableExpression + logicOp + tableExpression | logicOp + tableExpression + logicOp | tableExpression + logicOp | logicOp + tableExpression | tableExpression;
LogicOp is 'AND' or 'OR'
Does this make sense?!
' OR SomeTable' - does this alone form a valid expression?
Dec 12, 2013 at 2:46 PM
ahh thank you very much, I am very new to this.

and yes good point on the logicexpression, I knew that needed work.

Thanks for your help.
Dec 12, 2013 at 3:53 PM
OK, I found grammar explorer (very helpful tool!) and used it to fix my reduce/reduce errors. But i am still getting the same IndexOutOfRange error as described above in the same place on the same string:
[a].[b] = "

the error is also reproducible in grammar explorer. here is the updated grammar:
[Language("QueryLanguage", "1.0", "A Query Language based on JET where clauses")]
    public class QueryGrammar : Grammar
    {

        /// <summary>
        /// Initializes a new instance of the <see cref="QueryGrammar"/> class.
        /// </summary>
        public QueryGrammar()
            : base(false)
        { // true means case sensitive
            GrammarComments = @"A Query Language based on JET where clauses. Case-insensitive.";

            // Terminals (Lexing)
            NumberLiteral number = new NumberLiteral("number");
            StringLiteral STRING = new StringLiteral("STRING", "\"");

            //Let's allow big integers (with unlimited number of digits):
            number.DefaultIntTypes = new TypeCode[] { TypeCode.Int32, TypeCode.Int64, NumberLiteral.TypeCodeBigInt };
            IdentifierTerminal Name = new IdentifierTerminal("Name");
            //var Name = TerminalFactory.CreateSqlExtIdentifier(this, "id_simple");
            CommentTerminal comment = new CommentTerminal("comment", "//", "\n", "\r");
            //comment must be added to NonGrammarTerminals list; it is not used directly in grammar rules,
            // so we add it to this list to let Scanner know that it is also a valid terminal. 
            NonGrammarTerminals.Add(comment);
            comment = new CommentTerminal("multilineComment", "/*", "*/");
            NonGrammarTerminals.Add(comment);

            ConstantTerminal CONSTANT = new ConstantTerminal("CONSTANT");
            CONSTANT.Add("NULL", null);


            // Non-Terminals (Parsing)
            NonTerminal query = new NonTerminal("Query");
            NonTerminal tableExpression = new NonTerminal("TableExpression");
            NonTerminal tableExpressions = new NonTerminal("TableExpressions");
            NonTerminal table = new NonTerminal("Table");
            NonTerminal column = new NonTerminal("Column");
            NonTerminal tableOperator = new NonTerminal("TableOperator");
            NonTerminal value = new NonTerminal("Value");
            NonTerminal logicOp = new NonTerminal("LogicOp");
            NonTerminal parameter = new NonTerminal("Parameter");
            NonTerminal list = new NonTerminal("List");
            NonTerminal enclosure = new NonTerminal("Enclosure");
            NonTerminal closure = new NonTerminal("Closure");
            NonTerminal logicExpression = new NonTerminal("logicExpression");
            NonTerminal queryExpression = new NonTerminal("queryExpression");
            NonTerminal betweenStmt = new NonTerminal("BetweenStmt");
            NonTerminal expList = new NonTerminal("ExpList");

            //keywords
            KeyTerm AND = ToTerm("AND");
            KeyTerm OR = ToTerm("OR");
            KeyTerm IN = ToTerm("IN");
            KeyTerm BETWEEN = ToTerm("BETWEEN");
            KeyTerm LIKE = ToTerm("LIKE");
            KeyTerm NOT = ToTerm("NOT");
            KeyTerm dot = ToTerm(".", "dot");
            KeyTerm comma = ToTerm(",", "comma");
            KeyTerm LeftSquareBrace = ToTerm("[", "LeftSquareBrace");
            KeyTerm RightSquareBrace = ToTerm("]", "RightSquareBrace");
            KeyTerm LeftCurlyBrace = ToTerm("{", "LeftSCurlyBrace");
            KeyTerm RightCurlyBrace = ToTerm("}", "RightCurlyBrace");
            KeyTerm LeftQuote = ToTerm("\"", "LeftQuote");
            KeyTerm RightQuote = ToTerm("\"", "RightQuote");
            
            //set precedence of operators.
            RegisterOperators(90, AND);
            RegisterOperators(80, OR);
            RegisterOperators(70, "=", ">", "<", "<>", ">=", "<=", "IN", "LIKE", "NOT LIKE", "IS", "IS NOT", "BETWEEN");

            MarkPunctuation(",", "(", ")", "[", "]", ".", "\"", "{", "}");

            logicExpression.Rule = tableExpression + logicOp + tableExpression | "(" + logicExpression + ")";
            //queryExpression.Rule = MakePlusRule(queryExpression,logicOp,logicExpression);
            tableExpression.Rule = table + dot + column + tableOperator + value;
            tableExpression.ErrorRule = SyntaxError + ";";
            betweenStmt.Rule = BETWEEN + value + "AND";
            tableOperator.Rule = ToTerm("=") | ">" | "<" | "<>" | ">=" | "<=" | "LIKE" | "IN" | "NOT LIKE" | "IS" | "IS NOT" | betweenStmt;
            value.Rule = number | parameter | STRING | CONSTANT | expList;
            enclosure.Rule = ToTerm("(") | Empty;
            closure.Rule = ToTerm(")") | Empty;
            parameter.Rule = LeftQuote + "{" + Name + "}" + "\"" | "{" + Name + "}" | "#" + "{" + Name + "}" + "#";
            logicOp.Rule = AND | OR;
            expList.Rule = "(" + list + ")";
            list.Rule = MakePlusRule(list, comma, value);
            table.Rule = LeftSquareBrace + Name + RightSquareBrace;
            table.ErrorRule = SyntaxError + ";";
            column.Rule = LeftSquareBrace + Name + RightSquareBrace;
            column.ErrorRule = SyntaxError + ";";
            query.Rule = MakePlusRule(query, logicOp, logicExpression);

            Root = query;
        }
    }
I'm sure I'm just missing something obvious.
Thanks again for your help, your project is very powerful, and useful!
Coordinator
Dec 12, 2013 at 6:17 PM
it's a bug, surprisingly nobody reported it before. I'm fixing it
Coordinator
Dec 12, 2013 at 6:22 PM
temp fix for you. Find file SourceStream.cs and modify SetNewPosition method as follows:
private void SetNewPosition(int newPosition) {
  if (newPosition < Position)
    throw new Exception(Resources.ErrCannotMoveBackInSource); 
  int p = Position; 
  int col = Location.Column;
  int line = Location.Line; 
  while(p <  newPosition) {
    if (p >= _textLength)     //!!!!! add this
      break;                       // and this
    var curr = _chars[p];
    switch (curr) {
      case '\n': line++; col = 0; break;
      case '\r': break; 
      case '\t': col = (col / _tabWidth + 1) * _tabWidth;     break;
      default: col++; break; 
    } //switch
    p++;
  }
  Location = new SourceLocation(p, line, col); 
}
Dec 12, 2013 at 6:36 PM
Edited Dec 12, 2013 at 6:37 PM
Thank you very much, glad I could help ;)

Unfortunately, I'm now getting an IndexOutOfRangeException thrown in scanner.cs on line 88:
 private void NextToken() {
      //1. Check if there are buffered tokens
      if(Context.BufferedTokens.Count > 0) {
        Context.CurrentToken = Context.BufferedTokens.Pop();
        return; 
      }
      //2. Skip whitespace.
      _grammar.SkipWhitespace(Context.Source);
      //3. That's the token start, calc location (line and column)
      Context.Source.Position = Context.Source.PreviewPosition; <---Error occurs here
      //4. Check for EOF
      if (Context.Source.EOF()) {
        Context.CurrentToken = new Token(_grammar.Eof, Context.Source.Location, string.Empty, _grammar.Eof.Name);;
        return; 
      }
      //5. Actually scan the source text and construct a new token
      ScanToken(); 
    }//method
I will debug it and see if I can pinpoint the problem.
Coordinator
Dec 12, 2013 at 6:51 PM
not happening to me. Try the latest version I just uploaded. What input are you using (source sample)?
Dec 12, 2013 at 7:03 PM
Ahh, thank you very much! yes the problem is gone.

Great work!
Sep 17, 2014 at 8:33 AM
Edited Sep 17, 2014 at 8:51 AM
Can anyone provide sample for Query Language?
[Language("QueryLanguage", "1.0", "A Query Language based on JET where clauses")]
And didn't find anything concrete about JET where clauses, it's about microsoft access sql? Tried different sql queries with square braces, without them but always got compile error.