This project is read-only.

Custom Lexer... Everything fine except keywords.

Topics: Developer Forum
Feb 18, 2015 at 12:40 PM
Hi, i'm creating a custom lexer ( based on the examples on the WPF branch ) for my editor and everything was fine till i tried to highlight the keywords, all the other operators, strings, etc. where ok, but i'm totally stuck with the keywords... The code looks ok to me and my debug shows that the lexer successfully detect each keyword but for some reason the style is not correctly applied... I hope someone can help me.

Here is my lexer code ( Check the State.Identifier case )

``` using System;
using System.Drawing;
using System.Collections.Generic;

namespace ScintillaNET.Lexers
{
public sealed class BBLexer : CustomLexer
{

    private const int STYLE_COMMENT = 1;
    private const int STYLE_NUMBER = 2;
    private const int STYLE_KEYWORD = 3;
    private const int STYLE_STRING = 4;
    private const int STYLE_OPERATOR = 6;
    private const int STYLE_IDENTIFIER = 7;

    private const int KEYWORDS_KEYWORDS = 0;
    private const int KEYWORDS_USER1 = 1;
    private const int KEYWORDS_USER2 = 2;
    private const int KEYWORDS_USER3 = 3;


    public override Dictionary<string, int> StyleNameMapping
    {
        get
        {
            return new Dictionary<string, int>()
            {
                { "Comment", STYLE_COMMENT },
                { "Number", STYLE_NUMBER },
                { "Keyword", STYLE_KEYWORD },
                { "Keyword1", STYLE_KEYWORD },
                { "String", STYLE_STRING },
                { "Operator", STYLE_OPERATOR },
                { "Identifier", STYLE_IDENTIFIER },

            };
        }
    }

    public override Dictionary<string, int> KeywordNameMapping
    {
        get
        {
            return new Dictionary<string, int>()
            {
                { "Keywords", KEYWORDS_KEYWORDS },
                { "User1", KEYWORDS_USER1 },
                { "User2", KEYWORDS_USER2 },
                { "User3", KEYWORDS_USER3 },
            };
        }
    }

    // Setup
    public override string LexerName { get { return "bb_script"; } }
    protected override bool EnableCompositeKeywords { get { return true; } }
    protected override bool LowerKeywords { get { return true; } }
    public BBLexer(Scintilla scintilla) : base(scintilla) { }


    protected override void Initialize()
    {
        EnsureCompositeKeywords(
            new KeyValuePair<int, int>(KEYWORDS_KEYWORDS, STYLE_KEYWORD),
            new KeyValuePair<int, int>(KEYWORDS_USER1, STYLE_KEYWORD),
            new KeyValuePair<int, int>(KEYWORDS_USER2, STYLE_KEYWORD),
            new KeyValuePair<int, int>(KEYWORDS_USER3, STYLE_KEYWORD)
        );
        base.Initialize();
        Scintilla.Indentation.SmartIndentType = SmartIndent.None;
    }

    private enum State : int
    {
        Unknown = STATE_UNKNOWN,
        Identifier,
        Number,
        String,
        Comment,
        Operator,
    }

    new private State CurrentState
    {
        get { return (State)base.CurrentState; }
        set { base.CurrentState = (int)value; }
    }

    protected override void Style()
    {

        StartStyling();

        while (!EndOfText)
        {
            switch (CurrentState)
            {
                case State.Unknown:

                    bool consumed = false;
                    switch (CurrentCharacter)
                    {
                        case ';':
                            CurrentState = State.Comment;
                            break;
                        case '"':
                            CurrentState = State.String;
                            break;

                        default:
                            if (IsDigit(CurrentCharacter) || (CurrentCharacter == '.' && IsDigit(NextCharacter)))
                            {
                                CurrentState = State.Number;
                            }
                            else if (IsOperator(CurrentCharacter))
                            {
                                CurrentState = State.Operator;                                    
                            }
                            else if (IsIdentifierStart(CurrentCharacter))
                            {
                                CurrentState = State.Identifier;
                            }
                            else if (IsWhitespace(CurrentCharacter))
                            {
                                ConsumeWhitespace();
                                consumed = true;
                            }
                            break;
                    }
                    if (!consumed)
                        Consume();
                    break;


                /*//////////////////////////////////////////////////////////////////
                 * Identifier
                 */////////////////////////////////////////////////////////////////                    
                case State.Identifier:
                    if (!IsIdentifier(CurrentCharacter))
                    {
                        string ident = GetRange(CurrentBasePosition, CurrentPosition).ToLower();
                        int style;

                        if (CompositeKeywords.TryGetValue(ident, out style))
                        {
                            Console.WriteLine(ident);
                            SetStyle(style);
                            //SetStyle(STYLE_IDENTIFIER);
                        }
                        else
                        {
                            SetStyle(STYLE_IDENTIFIER);                                
                            CurrentState = State.Unknown;
                        }
                    }
                    else
                        Consume();

                    break;


                /*//////////////////////////////////////////////////////////////////
                 * Numbers
                 *//////////////////////////////////////////////////////////////////
                case State.Number:
                    if (!IsNumber(CurrentCharacter) )
                    {
                        SetStyle(STYLE_NUMBER);
                        CurrentState = State.Unknown;
                    }
                    else
                        Consume();
                    break;

                /*//////////////////////////////////////////////////////////////////
                 * Strings
                 *//////////////////////////////////////////////////////////////////
                case State.String:
                    if (CurrentCharacter == '"')
                    {
                        if (NextCharacter == '"')
                        {
                            Consume(2);
                        }
                        else
                        {
                            Consume();
                            SetStyle(STYLE_STRING);
                            CurrentState = State.Unknown;
                        }
                    }
                    else if (IsEndOfLine(CurrentCharacter))
                    {
                        MarkSyntaxError(CurrentBasePosition, CurrentPosition - CurrentBasePosition);
                        SetStyle(STYLE_STRING);
                        CurrentState = State.Unknown;
                    }
                    else
                        Consume();
                    break;

                /*//////////////////////////////////////////////////////////////////
                 * Operators
                 *//////////////////////////////////////////////////////////////////
                case State.Operator:
                    if (!IsOperator(CurrentCharacter))
                    {
                        SetStyle(STYLE_OPERATOR);
                        CurrentState = State.Unknown;
                    }
                    else
                        Consume();
                    break;

                /*//////////////////////////////////////////////////////////////////
                 * Commented Lines
                 *//////////////////////////////////////////////////////////////////
                case State.Comment: 
                    if (IsEndOfLine(CurrentCharacter))
                    {
                        Consume();
                        SetStyle(STYLE_COMMENT);
                        CurrentState = State.Unknown;
                    }
                    else
                        Consume();
                    break;

                default:
                    throw new Exception("Unknown State!");
            }
        }

    }
}
}
Feb 18, 2015 at 10:28 PM
Edited Feb 18, 2015 at 10:28 PM
Ah, there's the issue:
if (CompositeKeywords.TryGetValue(ident, out style))
{
    Console.WriteLine(ident);
    SetStyle(style);
    //SetStyle(STYLE_IDENTIFIER);
}
else
{
    SetStyle(STYLE_IDENTIFIER);
    CurrentState = State.Unknown;
}
You never leave the Identifier state.

It should instead read:
if (CompositeKeywords.TryGetValue(ident, out style))
{
    Console.WriteLine(ident);
    SetStyle(style);
    //SetStyle(STYLE_IDENTIFIER);
}
else
{
    SetStyle(STYLE_IDENTIFIER);
}
CurrentState = State.Unknown;
Feb 18, 2015 at 11:40 PM
Edited Feb 18, 2015 at 11:53 PM
Thanks for the help, but the code is still not working... With your change is almost the same process of any other state but the style persist with the problem... aw...

I tried with the following code without lucky.
case State.Identifier:
    if (!IsIdentifier(CurrentCharacter))
    {
        string ident = GetRange(CurrentBasePosition, CurrentPosition).ToLower();
        int style;

        if (CompositeKeywords.TryGetValue(ident, out style))
        {
            Console.WriteLine(ident); // Dbug
            SetStyle(style);                                
        }
        else
        {
            SetStyle(STYLE_IDENTIFIER);
        }
        CurrentState = State.Unknown;
    }
    else
        Consume();

    break;
Feb 21, 2015 at 1:16 AM
I just tested with the WPF branch, and it worked just fine for me. The C# code for the lexer is:
using System;
using System.Drawing;
using System.Collections.Generic;

namespace ScintillaNET.Lexers
{
    public sealed class BBLexer : CustomLexer
    {

        private const int STYLE_COMMENT = 1;
        private const int STYLE_NUMBER = 2;
        private const int STYLE_KEYWORD = 3;
        private const int STYLE_STRING = 4;
        private const int STYLE_OPERATOR = 6;
        private const int STYLE_IDENTIFIER = 7;

        private const int KEYWORDS_KEYWORDS = 0;
        private const int KEYWORDS_USER1 = 1;
        private const int KEYWORDS_USER2 = 2;
        private const int KEYWORDS_USER3 = 3;


        public override Dictionary<string, int> StyleNameMapping
        {
            get
            {
                return new Dictionary<string, int>()
            {
                { "Comment", STYLE_COMMENT },
                { "Number", STYLE_NUMBER },
                { "Keyword", STYLE_KEYWORD },
                { "Keyword1", STYLE_KEYWORD },
                { "String", STYLE_STRING },
                { "Operator", STYLE_OPERATOR },
                { "Identifier", STYLE_IDENTIFIER },

            };
            }
        }

        public override Dictionary<string, int> KeywordNameMapping
        {
            get
            {
                return new Dictionary<string, int>()
            {
                { "Keywords", KEYWORDS_KEYWORDS },
                { "User1", KEYWORDS_USER1 },
                { "User2", KEYWORDS_USER2 },
                { "User3", KEYWORDS_USER3 },
            };
            }
        }

        // Setup
        public override string LexerName { get { return "bb_script"; } }
        protected override bool EnableCompositeKeywords { get { return true; } }
        protected override bool LowerKeywords { get { return true; } }
        public BBLexer(Scintilla scintilla) : base(scintilla) { }


        protected override void Initialize()
        {
            EnsureCompositeKeywords(
                new KeyValuePair<int, int>(KEYWORDS_KEYWORDS, STYLE_KEYWORD),
                new KeyValuePair<int, int>(KEYWORDS_USER1, STYLE_KEYWORD),
                new KeyValuePair<int, int>(KEYWORDS_USER2, STYLE_KEYWORD),
                new KeyValuePair<int, int>(KEYWORDS_USER3, STYLE_KEYWORD)
            );
            base.Initialize();
            Scintilla.Indentation.SmartIndentType = SmartIndent.None;
        }

        private enum State : int
        {
            Unknown = STATE_UNKNOWN,
            Identifier,
            Number,
            String,
            Comment,
            Operator,
        }

        new private State CurrentState
        {
            get { return (State)base.CurrentState; }
            set { base.CurrentState = (int)value; }
        }

        protected override void Style()
        {

            StartStyling();

            while (!EndOfText)
            {
                switch (CurrentState)
                {
                    case State.Unknown:

                        bool consumed = false;
                        switch (CurrentCharacter)
                        {
                            case ';':
                                CurrentState = State.Comment;
                                break;
                            case '"':
                                CurrentState = State.String;
                                break;

                            default:
                                if (IsDigit(CurrentCharacter) || (CurrentCharacter == '.' && IsDigit(NextCharacter)))
                                {
                                    CurrentState = State.Number;
                                }
                                else if (IsOperator(CurrentCharacter))
                                {
                                    CurrentState = State.Operator;
                                }
                                else if (IsIdentifierStart(CurrentCharacter))
                                {
                                    CurrentState = State.Identifier;
                                }
                                else if (IsWhitespace(CurrentCharacter))
                                {
                                    ConsumeWhitespace();
                                    consumed = true;
                                }
                                break;
                        }
                        if (!consumed)
                            Consume();
                        break;


                    /*//////////////////////////////////////////////////////////////////
                     * Identifier
                     */
                    ////////////////////////////////////////////////////////////////                    
                    case State.Identifier:
                        if (!IsIdentifier(CurrentCharacter))
                        {
                            string ident = GetRange(CurrentBasePosition, CurrentPosition).ToLower();
                            int style;

                            if (CompositeKeywords.TryGetValue(ident, out style))
                            {
                                Console.WriteLine(ident);
                                SetStyle(style);
                                //SetStyle(STYLE_IDENTIFIER);
                            }
                            else
                            {
                                SetStyle(STYLE_IDENTIFIER);
                            }
                            CurrentState = State.Unknown;
                        }
                        else
                            Consume();

                        break;


                    /*//////////////////////////////////////////////////////////////////
                     * Numbers
                     */
                    /////////////////////////////////////////////////////////////////
                    case State.Number:
                        if (!IsNumber(CurrentCharacter))
                        {
                            SetStyle(STYLE_NUMBER);
                            CurrentState = State.Unknown;
                        }
                        else
                            Consume();
                        break;

                    /*//////////////////////////////////////////////////////////////////
                     * Strings
                     */
                    /////////////////////////////////////////////////////////////////
                    case State.String:
                        if (CurrentCharacter == '"')
                        {
                            if (NextCharacter == '"')
                            {
                                Consume(2);
                            }
                            else
                            {
                                Consume();
                                SetStyle(STYLE_STRING);
                                CurrentState = State.Unknown;
                            }
                        }
                        else if (IsEndOfLine(CurrentCharacter))
                        {
                            MarkSyntaxError(CurrentBasePosition, CurrentPosition - CurrentBasePosition);
                            SetStyle(STYLE_STRING);
                            CurrentState = State.Unknown;
                        }
                        else
                            Consume();
                        break;

                    /*//////////////////////////////////////////////////////////////////
                     * Operators
                     */
                    /////////////////////////////////////////////////////////////////
                    case State.Operator:
                        if (!IsOperator(CurrentCharacter))
                        {
                            SetStyle(STYLE_OPERATOR);
                            CurrentState = State.Unknown;
                        }
                        else
                            Consume();
                        break;

                    /*//////////////////////////////////////////////////////////////////
                     * Commented Lines
                     */
                    /////////////////////////////////////////////////////////////////
                    case State.Comment:
                        if (IsEndOfLine(CurrentCharacter))
                        {
                            Consume();
                            SetStyle(STYLE_COMMENT);
                            CurrentState = State.Unknown;
                        }
                        else
                            Consume();
                        break;

                    default:
                        throw new Exception("Unknown State!");
                }
            }

        }
    }
}
And the XML for the configuration (I borrowed VBScript's keyword list to test with) is:
<?xml version="1.0" encoding="utf-8"?>
<ScintillaNET xmlns="http://scintillanet.codeplex.com/LanguageConfiguration.xsd">
  <Language Name="bb_script">
    <Indentation TabWidth="4" />

    <Lexer LineCommentPrefix=";">
      <Keywords Name="Keywords" Inherit="False">
        addhandler addressof andalso alias and ansi as assembly attribute auto begin boolean byref byte byval call case catch
        cbool cbyte cchar cdate cdec cdbl char cint class clng cobj compare const continue cshort csng cstr ctype currency date
        decimal declare default delegate dim do double each else elseif end enum erase error event exit explicit false finally
        for friend function get gettype global gosub goto handles if implement implements imports in inherits integer interface
        is let lib like load long loop lset me mid mod module mustinherit mustoverride mybase myclass namespace new next not
        nothing notinheritable notoverridable object on option optional or orelse overloads overridable overrides paramarray
        preserve private property protected public raiseevent readonly redim rem removehandler rset resume return select set
        shadows shared short single static step stop string structure sub synclock then throw to true try type typeof unload
        unicode until variant wend when while with withevents writeonly xor
      </Keywords>
    </Lexer>
  </Language>
</ScintillaNET>
Which branch of ScintillaNET are you using this with?