[geany/geany] e88c03: Update Scintilla to version 3.3.1

Branch: refs/heads/master Author: Colomban Wendling ban@herbesfolles.org Committer: Colomban Wendling ban@herbesfolles.org Date: Sat, 27 Apr 2013 16:29:44 UTC Commit: e88c03d266e1b8d4279a76481d03c54a76fd961d https://github.com/geany/geany/commit/e88c03d266e1b8d4279a76481d03c54a76fd96...

Log Message: ----------- Update Scintilla to version 3.3.1

Modified Paths: -------------- scintilla/Makefile.am scintilla/gtk/PlatGTK.cxx scintilla/gtk/ScintillaGTK.cxx scintilla/include/ILexer.h scintilla/include/Platform.h scintilla/include/SciLexer.h scintilla/include/Scintilla.h scintilla/include/Scintilla.iface scintilla/lexers/LexCPP.cxx scintilla/lexers/LexHTML.cxx scintilla/lexers/LexHaskell.cxx scintilla/lexers/LexLaTeX.cxx scintilla/lexers/LexMatlab.cxx scintilla/lexers/LexOthers.cxx scintilla/lexers/LexPO.cxx scintilla/lexers/LexRuby.cxx scintilla/lexlib/LexAccessor.h scintilla/lexlib/LexerBase.cxx scintilla/lexlib/LexerModule.cxx scintilla/lexlib/StyleContext.h scintilla/lexlib/SubStyles.h scintilla/makefile.win32 scintilla/src/AutoComplete.cxx scintilla/src/AutoComplete.h scintilla/src/CellBuffer.cxx scintilla/src/CellBuffer.h scintilla/src/ContractionState.cxx scintilla/src/Document.cxx scintilla/src/Document.h scintilla/src/Editor.cxx scintilla/src/Editor.h scintilla/src/LineMarker.cxx scintilla/src/PositionCache.h scintilla/src/RunStyles.h scintilla/src/ScintillaBase.cxx scintilla/src/ScintillaBase.h scintilla/src/UniConversion.h scintilla/src/ViewStyle.cxx scintilla/src/ViewStyle.h scintilla/version.txt

Modified: scintilla/Makefile.am 1 files changed, 1 insertions(+), 0 deletions(-) =================================================================== @@ -22,6 +22,7 @@ lexers/LexForth.cxx \ lexers/LexFortran.cxx \ lexers/LexHTML.cxx \ lexers/LexHaskell.cxx \ +lexers/LexLaTeX.cxx \ lexers/LexLisp.cxx \ lexers/LexLua.cxx \ lexers/LexMarkdown.cxx \

Modified: scintilla/gtk/PlatGTK.cxx 15 files changed, 13 insertions(+), 2 deletions(-) =================================================================== @@ -105,9 +105,11 @@ struct LOGFONT { static GMutex *fontMutex = NULL;

static void InitializeGLIBThreads() { +#if !GLIB_CHECK_VERSION(2,31,0) if (!g_thread_supported()) { g_thread_init(NULL); } +#endif } #endif

@@ -115,7 +117,12 @@ static void FontMutexAllocate() { #if USE_LOCK if (!fontMutex) { InitializeGLIBThreads(); +#if GLIB_CHECK_VERSION(2,31,0) + fontMutex = g_new(GMutex, 1); + g_mutex_init(fontMutex); +#else fontMutex = g_mutex_new(); +#endif } #endif } @@ -123,7 +130,12 @@ static void FontMutexAllocate() { static void FontMutexFree() { #if USE_LOCK if (fontMutex) { +#if GLIB_CHECK_VERSION(2,31,0) + g_mutex_clear(fontMutex); + g_free(fontMutex); +#else g_mutex_free(fontMutex); +#endif fontMutex = NULL; } #endif @@ -143,8 +155,7 @@ static void FontMutexUnlock() { #endif }

-// On GTK+ 1.x holds a GdkFont* but on GTK+ 2.x can hold a GdkFont* or a -// PangoFontDescription*. +// Holds a PangoFontDescription*. class FontHandle { XYPOSITION width[128]; encodingType et;

Modified: scintilla/gtk/ScintillaGTK.cxx 21 files changed, 15 insertions(+), 6 deletions(-) =================================================================== @@ -296,7 +296,7 @@ class ScintillaGTK : public ScintillaBase { static gboolean TimeOut(ScintillaGTK *sciThis); static gboolean IdleCallback(ScintillaGTK *sciThis); static gboolean StyleIdle(ScintillaGTK *sciThis); - virtual void QueueStyling(int upTo); + virtual void QueueIdleWork(WorkNeeded::workItems items, int upTo); static void PopUpCB(GtkMenuItem *menuItem, ScintillaGTK *sciThis);

#if GTK_CHECK_VERSION(3,0,0) @@ -1138,6 +1138,7 @@ void ScintillaGTK::ScrollText(int linesToMove) { //Platform::DebugPrintf("ScintillaGTK::ScrollText %d %d %0d,%0d %0d,%0d\n", linesToMove, diff, // rc.left, rc.top, rc.right, rc.bottom); GtkWidget *wi = PWidget(wText); + NotifyUpdateUI();

if (IS_WIDGET_REALIZED(wi)) { gdk_window_scroll(WindowFromWidget(wi), 0, -diff); @@ -2740,7 +2741,9 @@ int ScintillaGTK::TimeOut(ScintillaGTK *sciThis) { gboolean ScintillaGTK::IdleCallback(ScintillaGTK *sciThis) { // Idler will be automatically stopped, if there is nothing // to do while idle. +#ifndef GDK_VERSION_3_6 gdk_threads_enter(); +#endif bool ret = sciThis->Idle(); if (ret == false) { // FIXME: This will remove the idler from GTK, we don't want to @@ -2748,23 +2751,29 @@ gboolean ScintillaGTK::IdleCallback(ScintillaGTK *sciThis) { // returns false (although, it should be harmless). sciThis->SetIdle(false); } +#ifndef GDK_VERSION_3_6 gdk_threads_leave(); +#endif return ret; }

gboolean ScintillaGTK::StyleIdle(ScintillaGTK *sciThis) { +#ifndef GDK_VERSION_3_6 gdk_threads_enter(); - sciThis->IdleStyling(); +#endif + sciThis->IdleWork(); +#ifndef GDK_VERSION_3_6 gdk_threads_leave(); +#endif // Idler will be automatically stopped return FALSE; }

-void ScintillaGTK::QueueStyling(int upTo) { - Editor::QueueStyling(upTo); - if (!styleNeeded.active) { +void ScintillaGTK::QueueIdleWork(WorkNeeded::workItems items, int upTo) { + Editor::QueueIdleWork(items, upTo); + if (!workNeeded.active) { // Only allow one style needed to be queued - styleNeeded.active = true; + workNeeded.active = true; g_idle_add_full(G_PRIORITY_HIGH_IDLE, reinterpret_cast<GSourceFunc>(StyleIdle), this, NULL); }

Modified: scintilla/include/ILexer.h 21 files changed, 19 insertions(+), 2 deletions(-) =================================================================== @@ -18,7 +18,7 @@ #define SCI_METHOD #endif

-enum { dvOriginal=0 }; +enum { dvOriginal=0, dvLineEnd=1 };

class IDocument { public: @@ -45,7 +45,12 @@ class IDocument { virtual int SCI_METHOD GetLineIndentation(int line) = 0; };

-enum { lvOriginal=0 }; +class IDocumentWithLineEnd : public IDocument { +public: + virtual int SCI_METHOD LineEnd(int line) const = 0; +}; + +enum { lvOriginal=0, lvSubStyles=1 };

class ILexer { public: @@ -62,6 +67,18 @@ class ILexer { virtual void * SCI_METHOD PrivateCall(int operation, void *pointer) = 0; };

+class ILexerWithSubStyles : public ILexer { +public: + virtual int SCI_METHOD LineEndTypesSupported() = 0; + virtual int SCI_METHOD AllocateSubStyles(int styleBase, int numberStyles) = 0; + virtual int SCI_METHOD SubStylesStart(int styleBase) = 0; + virtual int SCI_METHOD SubStylesLength(int styleBase) = 0; + virtual void SCI_METHOD FreeSubStyles() = 0; + virtual void SCI_METHOD SetIdentifiers(int style, const char *identifiers) = 0; + virtual int SCI_METHOD DistanceToSecondaryStyles() = 0; + virtual const char * SCI_METHOD GetSubStyleBases() = 0; +}; + class ILoader { public: virtual int SCI_METHOD Release() = 0;

Modified: scintilla/include/Platform.h 8 files changed, 4 insertions(+), 4 deletions(-) =================================================================== @@ -22,7 +22,7 @@ #define PLAT_WX 0 #define PLAT_QT 0 #define PLAT_FOX 0 -#define PLAT_NCURSES 0 +#define PLAT_CURSES 0

#if defined(FOX) #undef PLAT_FOX @@ -32,9 +32,9 @@ #undef PLAT_WX #define PLAT_WX 1

-#elif defined(NCURSES) -#undef PLAT_NCURSES -#define PLAT_NCURSES 1 +#elif defined(CURSES) +#undef PLAT_CURSES +#define PLAT_CURSES 1

#elif defined(SCINTILLA_QT) #undef PLAT_QT

Modified: scintilla/include/SciLexer.h 4 files changed, 4 insertions(+), 0 deletions(-) =================================================================== @@ -477,6 +477,7 @@ #define SCE_ERR_TIDY 19 #define SCE_ERR_JAVA_STACK 20 #define SCE_ERR_VALUE 21 +#define SCE_ERR_GCC_INCLUDED_FROM 22 #define SCE_BAT_DEFAULT 0 #define SCE_BAT_COMMENT 1 #define SCE_BAT_WORD 2 @@ -1019,6 +1020,8 @@ #define SCE_HA_COMMENTBLOCK 14 #define SCE_HA_COMMENTBLOCK2 15 #define SCE_HA_COMMENTBLOCK3 16 +#define SCE_HA_PRAGMA 17 +#define SCE_HA_PREPROCESSOR 18 #define SCE_T3_DEFAULT 0 #define SCE_T3_X_DEFAULT 1 #define SCE_T3_PREPROCESSOR 2 @@ -1345,6 +1348,7 @@ #define SCE_MYSQL_USER2 19 #define SCE_MYSQL_USER3 20 #define SCE_MYSQL_HIDDENCOMMAND 21 +#define SCE_MYSQL_PLACEHOLDER 22 #define SCE_PO_DEFAULT 0 #define SCE_PO_COMMENT 1 #define SCE_PO_MSGID 2

Modified: scintilla/include/Scintilla.h 26 files changed, 26 insertions(+), 0 deletions(-) =================================================================== @@ -148,6 +148,7 @@ #define SCI_MARKERDEFINEPIXMAP 2049 #define SCI_MARKERADDSET 2466 #define SCI_MARKERSETALPHA 2476 +#define SC_MAX_MARGIN 4 #define SC_MARGIN_SYMBOL 0 #define SC_MARGIN_NUMBER 1 #define SC_MARGIN_BACK 2 @@ -381,6 +382,7 @@ #define SCI_POSITIONFROMLINE 2167 #define SCI_LINESCROLL 2168 #define SCI_SCROLLCARET 2169 +#define SCI_SCROLLRANGE 2569 #define SCI_REPLACESEL 2170 #define SCI_SETREADONLY 2171 #define SCI_NULL 2172 @@ -696,6 +698,11 @@ #define SC_CASEINSENSITIVEBEHAVIOUR_IGNORECASE 1 #define SCI_AUTOCSETCASEINSENSITIVEBEHAVIOUR 2634 #define SCI_AUTOCGETCASEINSENSITIVEBEHAVIOUR 2635 +#define SC_ORDER_PRESORTED 0 +#define SC_ORDER_PERFORMSORT 1 +#define SC_ORDER_CUSTOM 2 +#define SCI_AUTOCSETORDER 2660 +#define SCI_AUTOCGETORDER 2661 #define SCI_ALLOCATE 2446 #define SCI_TARGETASUTF8 2447 #define SCI_SETLENGTHFORENCODE 2448 @@ -775,6 +782,8 @@ #define SCI_ANNOTATIONGETVISIBLE 2549 #define SCI_ANNOTATIONSETSTYLEOFFSET 2550 #define SCI_ANNOTATIONGETSTYLEOFFSET 2551 +#define SCI_RELEASEALLEXTENDEDSTYLES 2552 +#define SCI_ALLOCATEEXTENDEDSTYLES 2553 #define UNDO_MAY_COALESCE 1 #define SCI_ADDUNDOACTION 2560 #define SCI_CHARPOSITIONFROMPOINT 2561 @@ -853,6 +862,8 @@ #define SCI_FINDINDICATORHIDE 2642 #define SCI_VCHOMEDISPLAY 2652 #define SCI_VCHOMEDISPLAYEXTEND 2653 +#define SCI_GETCARETLINEVISIBLEALWAYS 2654 +#define SCI_SETCARETLINEVISIBLEALWAYS 2655 #define SCI_STARTRECORD 3001 #define SCI_STOPRECORD 3002 #define SCI_SETLEXER 4001 @@ -957,6 +968,21 @@ #define SCN_AUTOCCANCELLED 2025 #define SCN_AUTOCCHARDELETED 2026 #define SCN_HOTSPOTRELEASECLICK 2027 +#ifndef SCI_DISABLE_PROVISIONAL +#define SC_LINE_END_TYPE_DEFAULT 0 +#define SC_LINE_END_TYPE_UNICODE 1 +#define SCI_SETLINEENDTYPESALLOWED 2656 +#define SCI_GETLINEENDTYPESALLOWED 2657 +#define SCI_GETLINEENDTYPESACTIVE 2658 +#define SCI_GETLINEENDTYPESSUPPORTED 4018 +#define SCI_ALLOCATESUBSTYLES 4020 +#define SCI_GETSUBSTYLESSTART 4021 +#define SCI_GETSUBSTYLESLENGTH 4022 +#define SCI_FREESUBSTYLES 4023 +#define SCI_SETIDENTIFIERS 4024 +#define SCI_DISTANCETOSECONDARYSTYLES 4025 +#define SCI_GETSUBSTYLEBASES 4026 +#endif /* --Autogenerated -- end of section automatically generated from Scintilla.iface */

/* These structures are defined to be exactly the same shape as the Win32

Modified: scintilla/include/Scintilla.iface 79 files changed, 79 insertions(+), 0 deletions(-) =================================================================== @@ -326,6 +326,8 @@ fun void MarkerAddSet=2466(int line, int set) # Set the alpha used for a marker that is drawn in the text area, not the margin. set void MarkerSetAlpha=2476(int markerNumber, int alpha)

+val SC_MAX_MARGIN=4 + enu MarginType=SC_MARGIN_ val SC_MARGIN_SYMBOL=0 val SC_MARGIN_NUMBER=1 @@ -922,6 +924,11 @@ fun void LineScroll=2168(int columns, int lines) # Ensure the caret is visible. fun void ScrollCaret=2169(,)

+# Scroll the argument positions and the range between them into view giving +# priority to the primary position then the secondary position. +# This may be used to make a search match visible. +fun void ScrollRange=2569(position secondary, position primary) + # Replace the selected text with the argument text. fun void ReplaceSel=2170(, string text)

@@ -1835,6 +1842,17 @@ set void AutoCSetCaseInsensitiveBehaviour=2634(int behaviour,) # Get auto-completion case insensitive behaviour. get int AutoCGetCaseInsensitiveBehaviour=2635(,)

+enu Ordering=SC_ORDER_ +val SC_ORDER_PRESORTED=0 +val SC_ORDER_PERFORMSORT=1 +val SC_ORDER_CUSTOM=2 + +# Set the way autocompletion lists are ordered. +set void AutoCSetOrder=2660(int order,) + +# Get the way autocompletion lists are ordered. +get int AutoCGetOrder=2661(,) + # Enlarge the document to a particular size of text bytes. fun void Allocate=2446(int bytes,)

@@ -2062,6 +2080,12 @@ set void AnnotationSetStyleOffset=2550(int style,) # Get the start of the range of style numbers used for annotations get int AnnotationGetStyleOffset=2551(,)

+# Release all extended (>255) style numbers +fun void ReleaseAllExtendedStyles=2552(,) + +# Allocate some extended (>255) style numbers and return the start of the range +fun int AllocateExtendedStyles=2553(int numberStyles,) + val UNDO_MAY_COALESCE=1

# Add a container action to the undo stack @@ -2267,6 +2291,12 @@ fun void VCHomeDisplay=2652(,) # Like VCHomeDisplay but extending selection to new caret position. fun void VCHomeDisplayExtend=2653(,)

+# Is the caret line always visible? +get bool GetCaretLineVisibleAlways=2654(,) + +# Sets the caret line to always visible. +set void SetCaretLineVisibleAlways=2655(bool alwaysVisible,) + # Start notifying the container of all key presses and commands. fun void StartRecord=3001(,)

@@ -2922,6 +2952,7 @@ val SCE_ERR_ABSF=18 val SCE_ERR_TIDY=19 val SCE_ERR_JAVA_STACK=20 val SCE_ERR_VALUE=21 +val SCE_ERR_GCC_INCLUDED_FROM=22 # Lexical states for SCLEX_BATCH lex Batch=SCLEX_BATCH SCE_BAT_ val SCE_BAT_DEFAULT=0 @@ -3405,6 +3436,7 @@ val SCE_KIX_FUNCTIONS=8 val SCE_KIX_OPERATOR=9 val SCE_KIX_IDENTIFIER=31 # Lexical states for SCLEX_GUI4CLI +lex Gui4Cli=SCLEX_GUI4CLI SCE_GC_ val SCE_GC_DEFAULT=0 val SCE_GC_COMMENTLINE=1 val SCE_GC_COMMENTBLOCK=2 @@ -3549,6 +3581,8 @@ val SCE_HA_COMMENTLINE=13 val SCE_HA_COMMENTBLOCK=14 val SCE_HA_COMMENTBLOCK2=15 val SCE_HA_COMMENTBLOCK3=16 +val SCE_HA_PRAGMA=17 +val SCE_HA_PREPROCESSOR=18 # Lexical states of SCLEX_TADS3 lex TADS3=SCLEX_TADS3 SCE_T3_ val SCE_T3_DEFAULT=0 @@ -3913,6 +3947,7 @@ val SCE_MYSQL_USER1=18 val SCE_MYSQL_USER2=19 val SCE_MYSQL_USER3=20 val SCE_MYSQL_HIDDENCOMMAND=21 +val SCE_MYSQL_PLACEHOLDER=22 # Lexical state for SCLEX_PO lex Po=SCLEX_PO SCE_PO_ val SCE_PO_DEFAULT=0 @@ -4242,6 +4277,50 @@ evt void AutoCCancelled=2025(void) evt void AutoCCharDeleted=2026(void) evt void HotSpotReleaseClick=2027(int modifiers, int position)

+cat Provisional + +# Line end types which may be used in addition to LF, CR, and CRLF +# SC_LINE_END_TYPE_UNICODE includes U+2028 Line Separator, +# U+2029 Paragraph Separator, and U+0085 Next Line +enu LineEndType=SC_LINE_END_TYPE_ +val SC_LINE_END_TYPE_DEFAULT=0 +val SC_LINE_END_TYPE_UNICODE=1 + +# Set the line end types that the application wants to use. May not be used if incompatible with lexer or encoding. +set void SetLineEndTypesAllowed=2656(int lineEndBitSet,) + +# Get the line end types currently allowed. +get int GetLineEndTypesAllowed=2657(,) + +# Get the line end types currently recognised. May be a subset of the allowed types due to lexer limitation. +get int GetLineEndTypesActive=2658(,) + +# Bit set of LineEndType enumertion for which line ends beyond the standard +# LF, CR, and CRLF are supported by the lexer. +get int GetLineEndTypesSupported=4018(,) + +# Allocate a set of sub styles for a particular base style, returning start of range +fun int AllocateSubStyles=4020(int styleBase, int numberStyles) + +# The starting style number for the sub styles associated with a base style +get int GetSubStylesStart=4021(int styleBase,) + +# The number of sub styles associated with a base style +get int GetSubStylesLength=4022(int styleBase,) + +# Free allocated sub styles +fun void FreeSubStyles=4023(,) + +# Set the identifiers that are shown in a particular style +set void SetIdentifiers=4024(int style, string identifiers) + +# Where styles are duplicated by a feature such as active/inactive code +# return the distance between the two types. +get int DistanceToSecondaryStyles=4025(,) + +# Get the set of base styles that can be extended with sub styles +get int GetSubStyleBases=4026(, stringresult styles) + cat Deprecated

# Deprecated in 2.21

Modified: scintilla/lexers/LexCPP.cxx 162 files changed, 121 insertions(+), 41 deletions(-) =================================================================== @@ -30,6 +30,7 @@ #include "LexerModule.h" #include "OptionSet.h" #include "SparseState.h" +#include "SubStyles.h"

#ifdef SCI_NAMESPACE using namespace Scintilla; @@ -65,9 +66,8 @@ static bool followsReturnKeyword(StyleContext &sc, LexAccessor &styler) { int pos = (int) sc.currentPos; int currentLine = styler.GetLine(pos); int lineStartPos = styler.LineStart(currentLine); - char ch; while (--pos > lineStartPos) { - ch = styler.SafeGetCharAt(pos); + char ch = styler.SafeGetCharAt(pos); if (ch != ' ' && ch != '\t') { break; } @@ -87,11 +87,15 @@ static std::string GetRestOfLine(LexAccessor &styler, int start, bool allowSpace std::string restOfLine; int i =0; char ch = styler.SafeGetCharAt(start, '\n'); - while ((ch != '\r') && (ch != '\n')) { + int endLine = styler.LineEnd(styler.GetLine(start)); + while (((start+i) < endLine) && (ch != '\r')) { + char chNext = styler.SafeGetCharAt(start + i + 1, '\n'); + if (ch == '/' && (chNext == '/' || chNext == '*')) + break; if (allowSpace || (ch != ' ')) restOfLine += ch; i++; - ch = styler.SafeGetCharAt(start + i, '\n'); + ch = chNext; } return restOfLine; } @@ -127,8 +131,9 @@ struct PPDefinition { int line; std::string key; std::string value; - PPDefinition(int line_, const std::string &key_, const std::string &value_) : - line(line_), key(key_), value(value_) { + bool isUndef; + PPDefinition(int line_, const std::string &key_, const std::string &value_, bool isUndef_ = false) : + line(line_), key(key_), value(value_), isUndef(isUndef_) { } };

@@ -307,7 +312,9 @@ struct OptionSetCPP : public OptionSet<OptionsCPP> { } };

-class LexerCPP : public ILexer { +static const char styleSubable[] = {SCE_C_IDENTIFIER, SCE_C_COMMENTDOCKEYWORD, 0}; + +class LexerCPP : public ILexerWithSubStyles { bool caseSensitive; CharacterSet setWord; CharacterSet setNegationOp; @@ -326,6 +333,8 @@ class LexerCPP : public ILexer { OptionSetCPP osCPP; SparseStatestd::string rawStringTerminators; enum { activeFlag = 0x40 }; + enum { ssIdentifier, ssDocKeyword }; + SubStyles subStyles; public: LexerCPP(bool caseSensitive_) : caseSensitive(caseSensitive_), @@ -333,7 +342,8 @@ class LexerCPP : public ILexer { setNegationOp(CharacterSet::setNone, "!"), setArithmethicOp(CharacterSet::setNone, "+-/*%"), setRelOp(CharacterSet::setNone, "=!<>"), - setLogicalOp(CharacterSet::setNone, "|&") { + setLogicalOp(CharacterSet::setNone, "|&"), + subStyles(styleSubable, 0x80, 0x40, activeFlag) { } virtual ~LexerCPP() { } @@ -341,7 +351,7 @@ class LexerCPP : public ILexer { delete this; } int SCI_METHOD Version() const { - return lvOriginal; + return lvSubStyles; } const char * SCI_METHOD PropertyNames() { return osCPP.PropertyNames(); @@ -364,6 +374,32 @@ class LexerCPP : public ILexer { return 0; }

+ int SCI_METHOD LineEndTypesSupported() { + return SC_LINE_END_TYPE_UNICODE; + }; + + int SCI_METHOD AllocateSubStyles(int styleBase, int numberStyles) { + return subStyles.Allocate(styleBase, numberStyles); + } + int SCI_METHOD SubStylesStart(int styleBase) { + return subStyles.Start(styleBase); + } + int SCI_METHOD SubStylesLength(int styleBase) { + return subStyles.Length(styleBase); + } + void SCI_METHOD FreeSubStyles() { + subStyles.Free(); + } + void SCI_METHOD SetIdentifiers(int style, const char *identifiers) { + subStyles.SetIdentifiers(style, identifiers); + } + int SCI_METHOD DistanceToSecondaryStyles() { + return activeFlag; + } + const char * SCI_METHOD GetSubStyleBases() { + return styleSubable; + } + static ILexer *LexerFactoryCPP() { return new LexerCPP(true); } @@ -457,6 +493,8 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle,

CharacterSet setWordStart(CharacterSet::setAlpha, "_", 0x80, true);

+ CharacterSet setInvalidRawFirst(CharacterSet::setNone, " )\\t\v\f\n"); + if (options.identifiersAllowDollars) { setWordStart.Add('$'); } @@ -468,6 +506,7 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle, bool continuationLine = false; bool isIncludePreprocessor = false; bool isStringInPreprocessor = false; + bool inRERange = false;

int lineCurrent = styler.GetLine(startPos); if ((MaskActive(initStyle) == SCE_C_PREPROCESSOR) || @@ -475,15 +514,10 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle, (MaskActive(initStyle) == SCE_C_COMMENTLINEDOC)) { // Set continuationLine if last character of previous line is '' if (lineCurrent > 0) { - int chBack = styler.SafeGetCharAt(startPos-1, 0); - int chBack2 = styler.SafeGetCharAt(startPos-2, 0); - int lineEndChar = '!'; - if (chBack2 == '\r' && chBack == '\n') { - lineEndChar = styler.SafeGetCharAt(startPos-3, 0); - } else if (chBack == '\n' || chBack == '\r') { - lineEndChar = chBack2; + int endLinePrevious = styler.LineEnd(lineCurrent - 1); + if (endLinePrevious > 0) { + continuationLine = styler.SafeGetCharAt(endLinePrevious-1) == '\'; } - continuationLine = lineEndChar == '\'; } }

@@ -497,7 +531,7 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle, } }

- StyleContext sc(startPos, length, initStyle, styler, 0x7f); + StyleContext sc(startPos, length, initStyle, styler, static_cast<char>(0xff)); LinePPState preproc = vlls.ForLine(lineCurrent);

bool definitionsChanged = false; @@ -515,7 +549,10 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle,

std::map<std::string, std::string> preprocessorDefinitions = preprocessorDefinitionsStart; for (std::vector<PPDefinition>::iterator itDef = ppDefineHistory.begin(); itDef != ppDefineHistory.end(); ++itDef) { - preprocessorDefinitions[itDef->key] = itDef->value; + if (itDef->isUndef) + preprocessorDefinitions.erase(itDef->key); + else + preprocessorDefinitions[itDef->key] = itDef->value; }

std::string rawStringTerminator = rawStringTerminators.ValueAt(lineCurrent-1); @@ -523,6 +560,11 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle,

int activitySet = preproc.IsInactive() ? activeFlag : 0;

+ const WordClassifier &classifierIdentifiers = subStyles.Classifier(SCE_C_IDENTIFIER); + const WordClassifier &classifierDocKeyWords = subStyles.Classifier(SCE_C_COMMENTDOCKEYWORD); + + int lineEndNext = styler.LineEnd(lineCurrent); + for (; sc.More();) {

if (sc.atLineStart) { @@ -541,6 +583,7 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle, visibleChars = 0; lastWordWasUUID = false; isIncludePreprocessor = false; + inRERange = false; if (preproc.IsInactive()) { activitySet = activeFlag; sc.SetState(sc.state | activitySet); @@ -549,6 +592,7 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle,

if (sc.atLineEnd) { lineCurrent++; + lineEndNext = styler.LineEnd(lineCurrent); vlls.Add(lineCurrent, preproc); if (rawStringTerminator != "") { rawSTNew.Set(lineCurrent-1, rawStringTerminator); @@ -557,11 +601,13 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle,

// Handle line continuation generically. if (sc.ch == '\') { - if (sc.chNext == '\n' || sc.chNext == '\r') { + if (static_cast<int>((sc.currentPos+1)) >= lineEndNext) { lineCurrent++; + lineEndNext = styler.LineEnd(lineCurrent); vlls.Add(lineCurrent, preproc); sc.Forward(); if (sc.ch == '\r' && sc.chNext == '\n') { + // Even in UTF-8, \r and \n are separate sc.Forward(); } continuationLine = true; @@ -586,7 +632,7 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle, } break; case SCE_C_IDENTIFIER: - if (!setWord.Contains(sc.ch) || (sc.ch == '.')) { + if (sc.atLineStart || sc.atLineEnd || !setWord.Contains(sc.ch) || (sc.ch == '.')) { char s[1000]; if (caseSensitive) { sc.GetCurrent(s, sizeof(s)); @@ -600,11 +646,16 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle, sc.ChangeState(SCE_C_WORD2|activitySet); } else if (keywords4.InList(s)) { sc.ChangeState(SCE_C_GLOBALCLASS|activitySet); + } else { + int subStyle = classifierIdentifiers.ValueFor(s); + if (subStyle >= 0) { + sc.ChangeState(subStyle|activitySet); + } } const bool literalString = sc.ch == '"'; if (literalString || sc.ch == ''') { size_t lenS = strlen(s); - const bool raw = literalString && sc.chPrev == 'R'; + const bool raw = literalString && sc.chPrev == 'R' && !setInvalidRawFirst.Contains(sc.chNext); if (raw) s[lenS--] = '\0'; bool valid = @@ -692,8 +743,15 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle, } else { sc.GetCurrentLowered(s, sizeof(s)); } - if (!IsASpace(sc.ch) || !keywords3.InList(s + 1)) { + if (!IsASpace(sc.ch)) { sc.ChangeState(SCE_C_COMMENTDOCKEYWORDERROR|activitySet); + } else if (!keywords3.InList(s + 1)) { + int subStyleCDKW = classifierDocKeyWords.ValueFor(s+1); + if (subStyleCDKW >= 0) { + sc.ChangeState(subStyleCDKW|activitySet); + } else { + sc.ChangeState(SCE_C_COMMENTDOCKEYWORDERROR|activitySet); + } } sc.SetState(styleBeforeDCKeyword|activitySet); } @@ -745,16 +803,18 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle, case SCE_C_REGEX: if (sc.atLineStart) { sc.SetState(SCE_C_DEFAULT|activitySet); - } else if (sc.ch == '/') { + } else if (! inRERange && sc.ch == '/') { sc.Forward(); while ((sc.ch < 0x80) && islower(sc.ch)) sc.Forward(); // gobble regex flags sc.SetState(SCE_C_DEFAULT|activitySet); - } else if (sc.ch == '\') { - // Gobble up the quoted character - if (sc.chNext == '\' || sc.chNext == '/') { - sc.Forward(); - } + } else if (sc.ch == '\' && (static_cast<int>(sc.currentPos+1) < lineEndNext)) { + // Gobble up the escaped character + sc.Forward(); + } else if (sc.ch == '[') { + inRERange = true; + } else if (sc.ch == ']') { + inRERange = false; } break; case SCE_C_STRINGEOL: @@ -780,7 +840,7 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle, } break; case SCE_C_UUID: - if (sc.ch == '\r' || sc.ch == '\n' || sc.ch == ')') { + if (sc.atLineEnd || sc.ch == ')') { sc.SetState(SCE_C_DEFAULT|activitySet); } } @@ -788,6 +848,7 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle, if (sc.atLineEnd && !atLineEndBeforeSwitch) { // State exit processing consumed characters up to end of line. lineCurrent++; + lineEndNext = styler.LineEnd(lineCurrent); vlls.Add(lineCurrent, preproc); }

@@ -809,7 +870,7 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle, } else { sc.SetState(SCE_C_NUMBER|activitySet); } - } else if (setWordStart.Contains(sc.ch) || (sc.ch == '@')) { + } else if (!sc.atLineEnd && (setWordStart.Contains(sc.ch) || (sc.ch == '@'))) { if (lastWordWasUUID) { sc.SetState(SCE_C_UUID|activitySet); lastWordWasUUID = false; @@ -835,6 +896,7 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle, && (!setCouldBePostOp.Contains(chPrevNonWhite) || !FollowsPostfixOperator(sc, styler))) { sc.SetState(SCE_C_REGEX|activitySet); // JavaScript's RegEx + inRERange = false; } else if (sc.ch == '"') { if (sc.chPrev == 'R') { styler.Flush(); @@ -934,10 +996,22 @@ void SCI_METHOD LexerCPP::Lex(unsigned int startPos, int length, int initStyle, } } } + } else if (sc.Match("undef")) { + if (options.updatePreprocessor && !preproc.IsInactive()) { + std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 5, true); + std::vectorstd::string tokens = Tokenize(restOfLine); + std::string key; + if (tokens.size() >= 1) { + key = tokens[0]; + preprocessorDefinitions.erase(key); + ppDefineHistory.push_back(PPDefinition(lineCurrent, key, "", true)); + definitionsChanged = true; + } + } } } } - } else if (isoperator(static_cast<char>(sc.ch))) { + } else if (isoperator(sc.ch)) { sc.SetState(SCE_C_OPERATOR|activitySet); } } @@ -968,10 +1042,12 @@ void SCI_METHOD LexerCPP::Fold(unsigned int startPos, int length, int initStyle,

unsigned int endPos = startPos + length; int visibleChars = 0; + bool inLineComment = false; int lineCurrent = styler.GetLine(startPos); int levelCurrent = SC_FOLDLEVELBASE; if (lineCurrent > 0) levelCurrent = styler.LevelAt(lineCurrent-1) >> 16; + unsigned int lineStartNext = styler.LineStart(lineCurrent+1); int levelMinCurrent = levelCurrent; int levelNext = levelCurrent; char chNext = styler[startPos]; @@ -984,11 +1060,13 @@ void SCI_METHOD LexerCPP::Fold(unsigned int startPos, int length, int initStyle, int stylePrev = style; style = styleNext; styleNext = MaskActive(styler.StyleAt(i + 1)); - bool atEOL = (ch == '\r' && chNext != '\n') || (ch == '\n'); - if (options.foldComment && options.foldCommentMultiline && IsStreamCommentStyle(style)) { - if (!IsStreamCommentStyle(stylePrev) && (stylePrev != SCE_C_COMMENTLINEDOC)) { + bool atEOL = i == (lineStartNext-1); + if ((style == SCE_C_COMMENTLINE) || (style == SCE_C_COMMENTLINEDOC)) + inLineComment = true; + if (options.foldComment && options.foldCommentMultiline && IsStreamCommentStyle(style) && !inLineComment) { + if (!IsStreamCommentStyle(stylePrev)) { levelNext++; - } else if (!IsStreamCommentStyle(styleNext) && (styleNext != SCE_C_COMMENTLINEDOC) && !atEOL) { + } else if (!IsStreamCommentStyle(styleNext) && !atEOL) { // Comments don't end at end of line and the next character may be unstyled. levelNext--; } @@ -1052,6 +1130,7 @@ void SCI_METHOD LexerCPP::Fold(unsigned int startPos, int length, int initStyle, styler.SetLevel(lineCurrent, lev); } lineCurrent++; + lineStartNext = styler.LineStart(lineCurrent+1); levelCurrent = levelNext; levelMinCurrent = levelCurrent; if (atEOL && (i == static_cast<unsigned int>(styler.Length()-1))) { @@ -1059,6 +1138,7 @@ void SCI_METHOD LexerCPP::Fold(unsigned int startPos, int length, int initStyle, styler.SetLevel(lineCurrent, (levelCurrent | levelCurrent << 16) | SC_FOLDLEVELWHITEFLAG); } visibleChars = 0; + inLineComment = false; } } } @@ -1172,7 +1252,7 @@ bool LexerCPP::EvaluateExpression(const std::string &expr, const std::map<std::s std::vectorstd::string tokens; const char *cp = expr.c_str(); for (;;) { - if (setWord.Contains(*cp)) { + if (setWord.Contains(static_cast<unsigned char>(*cp))) { word += *cp; } else { std::map<std::string, std::string>::const_iterator it = preprocessorDefinitions.find(word); @@ -1187,13 +1267,13 @@ bool LexerCPP::EvaluateExpression(const std::string &expr, const std::map<std::s } if ((*cp != ' ') && (*cp != '\t')) { std::string op(cp, 1); - if (setRelOp.Contains(*cp)) { - if (setRelOp.Contains(cp[1])) { + if (setRelOp.Contains(static_cast<unsigned char>(*cp))) { + if (setRelOp.Contains(static_cast<unsigned char>(cp[1]))) { op += cp[1]; cp++; } - } else if (setLogicalOp.Contains(*cp)) { - if (setLogicalOp.Contains(cp[1])) { + } else if (setLogicalOp.Contains(static_cast<unsigned char>(*cp))) { + if (setLogicalOp.Contains(static_cast<unsigned char>(cp[1]))) { op += cp[1]; cp++; }

Modified: scintilla/lexers/LexHTML.cxx 15 files changed, 4 insertions(+), 11 deletions(-) =================================================================== @@ -445,11 +445,6 @@ static int StateForScript(script_type scriptLanguage) { return Result; }

-static inline bool ishtmlwordchar(int ch) { - return !isascii(ch) || - (isalnum(ch) || ch == '.' || ch == '-' || ch == '_' || ch == ':' || ch == '!' || ch == '#'); -} - static inline bool issgmlwordchar(int ch) { return !isascii(ch) || (isalnum(ch) || ch == '.' || ch == '_' || ch == ':' || ch == '!' || ch == '#' || ch == '['); @@ -484,10 +479,6 @@ static bool isLineEnd(int ch) { return ch == '\r' || ch == '\n'; }

-static bool isOKBeforeRE(int ch) { - return (ch == '(') || (ch == '=') || (ch == ','); -} - static bool isMakoBlockEnd(const int ch, const int chNext, const char *blockType) { if (strlen(blockType) == 0) { return ((ch == '%') && (chNext == '>')); @@ -691,6 +682,8 @@ static void ColouriseHyperTextDoc(unsigned int startPos, int length, int initSty const CharacterSet setHTMLWord(CharacterSet::setAlphaNum, ".-_:!#", 0x80, true); const CharacterSet setTagContinue(CharacterSet::setAlphaNum, ".-_:!#[", 0x80, true); const CharacterSet setAttributeContinue(CharacterSet::setAlphaNum, ".-_:!#/", 0x80, true); + // TODO: also handle + and - (except if they're part of ++ or --) and return keywords + const CharacterSet setOKBeforeJSRE(CharacterSet::setNone, "([{=,:;!%^&*|?~");

int levelPrev = styler.LevelAt(lineCurrent) & SC_FOLDLEVELNUMBERMASK; int levelCurrent = levelPrev; @@ -905,7 +898,7 @@ static void ColouriseHyperTextDoc(unsigned int startPos, int length, int initSty !IsScriptCommentState(state)) { beforeLanguage = scriptLanguage; scriptLanguage = segIsScriptingIndicator(styler, i + 2, i + 6, isXml ? eScriptXML : eScriptPHP); - if (scriptLanguage != eScriptPHP && isStringState(state)) continue; + if ((scriptLanguage != eScriptPHP) && (isStringState(state) || (state==SCE_H_COMMENT))) continue; styler.ColourTo(i - 1, StateToPrint); beforePreProc = state; i++; @@ -1592,7 +1585,7 @@ static void ColouriseHyperTextDoc(unsigned int startPos, int length, int initSty } else if (ch == '/' && chNext == '/') { styler.ColourTo(i - 1, StateToPrint); state = SCE_HJ_COMMENTLINE; - } else if (ch == '/' && isOKBeforeRE(chPrevNonWhite)) { + } else if (ch == '/' && setOKBeforeJSRE.Contains(chPrevNonWhite)) { styler.ColourTo(i - 1, StateToPrint); state = SCE_HJ_REGEX; } else if (ch == '"') {

Modified: scintilla/lexers/LexHaskell.cxx 351 files changed, 209 insertions(+), 142 deletions(-) =================================================================== @@ -9,6 +9,8 @@ * * Several bug fixes by Krasimir Angelov - kr.angelov at gmail.com * + * Improvements by kudah - kudahkukarek at gmail.com + * * TODO: * * Implement a folder :) * * Nice Character-lexing (stuff inside '''), LexPython has @@ -57,22 +59,24 @@ #define HA_MODE_FFI 5 #define HA_MODE_TYPE 6

-static inline bool IsNewline(const int ch) { - return (ch == '\n' || ch == '\r'); -} - -static inline bool IsWhitespace(const int ch) { - return ( ch == ' ' - || ch == '\t' - || IsNewline(ch) ); +static inline bool IsAWordStart(const int ch) { + return (IsLowerCase(ch) || IsUpperCase(ch) || ch == '_'); }

-static inline bool IsAWordStart(const int ch) { - return (ch < 0x80) && (isalnum(ch) || ch == '_'); +static inline bool IsAWordChar(const int ch, const bool magicHash) { + return ( IsAlphaNumeric(ch) + || ch == '_' + || ch == ''' + || (magicHash && ch == '#')); }

-static inline bool IsAWordChar(const int ch) { - return (ch < 0x80) && (isalnum(ch) || ch == '.' || ch == '_' || ch == '''); +static inline bool IsAnOperatorChar(const int ch) { + return + ( ch == '!' || ch == '#' || ch == '$' || ch == '%' + || ch == '&' || ch == '*' || ch == '+' || ch == '-' + || ch == '.' || ch == '/' || ch == ':' || ch == '<' + || ch == '=' || ch == '>' || ch == '?' || ch == '@' + || ch == '\' || ch == '^' || ch == '|' || ch == '~'); }

static void ColorizeHaskellDoc(unsigned int startPos, int length, int initStyle, @@ -81,230 +85,293 @@ static void ColorizeHaskellDoc(unsigned int startPos, int length, int initStyle, WordList &keywords = *keywordlists[0]; WordList &ffi = *keywordlists[1];

+ // property lexer.haskell.allow.hash + // Set to 1 to allow the # character in identifiers with the haskell lexer. + // (GHC -XMagicHash extension) + const bool magicHash = styler.GetPropertyInt("lexer.haskell.allow.hash") != 0; + const bool stylingWithinPreprocessor = styler.GetPropertyInt("styling.within.preprocessor") != 0; + StyleContext sc(startPos, length, initStyle, styler);

int lineCurrent = styler.GetLine(startPos); int state = lineCurrent ? styler.GetLineState(lineCurrent-1) : HA_MODE_DEFAULT; int mode = state & 0xF; - int xmode = state >> 4; + int xmode = state >> 4; // obscure parameter. Means different things in different modes.

while (sc.More()) { // Check for state end

// Operator if (sc.state == SCE_HA_OPERATOR) { - if (isascii(sc.ch) && isoperator(static_cast<char>(sc.ch))) { - sc.Forward(); - } else { - styler.ColourTo(sc.currentPos - 1, sc.state); - sc.ChangeState(SCE_HA_DEFAULT); + int style = SCE_HA_OPERATOR; + + if (sc.ch == ':' && + // except "::" + !(sc.chNext == ':' && !IsAnOperatorChar(sc.GetRelative(2)))) { + style = SCE_HA_CAPITAL; } + + while(IsAnOperatorChar(sc.ch)) + sc.Forward(); + + styler.ColourTo(sc.currentPos - 1, style); + sc.ChangeState(SCE_HA_DEFAULT); } // String else if (sc.state == SCE_HA_STRING) { if (sc.ch == '"') { - sc.Forward(); - styler.ColourTo(sc.currentPos-1, sc.state); - sc.ChangeState(SCE_HA_DEFAULT); + sc.Forward(); + sc.SetState(SCE_HA_DEFAULT); } else if (sc.ch == '\') { sc.Forward(2); } else if (sc.atLineEnd) { - styler.ColourTo(sc.currentPos-1, sc.state); - sc.ChangeState(SCE_HA_DEFAULT); - } else { - sc.Forward(); - } + sc.SetState(SCE_HA_DEFAULT); + } else { + sc.Forward(); + } } // Char else if (sc.state == SCE_HA_CHARACTER) { if (sc.ch == ''') { - sc.Forward(); - styler.ColourTo(sc.currentPos-1, sc.state); - sc.ChangeState(SCE_HA_DEFAULT); + sc.Forward(); + sc.SetState(SCE_HA_DEFAULT); } else if (sc.ch == '\') { sc.Forward(2); } else if (sc.atLineEnd) { - styler.ColourTo(sc.currentPos-1, sc.state); - sc.ChangeState(SCE_HA_DEFAULT); - } else { - sc.Forward(); - } + sc.SetState(SCE_HA_DEFAULT); + } else { + sc.Forward(); + } } // Number else if (sc.state == SCE_HA_NUMBER) { - if (IsADigit(sc.ch, xmode)) { + if (IsADigit(sc.ch, xmode) || + (sc.ch=='.' && IsADigit(sc.chNext, xmode))) { sc.Forward(); } else if ((xmode == 10) && (sc.ch == 'e' || sc.ch == 'E') && (IsADigit(sc.chNext) || sc.chNext == '+' || sc.chNext == '-')) { - sc.Forward(); - if (sc.ch == '+' || sc.ch == '-') - sc.Forward(); + sc.Forward(); + if (sc.ch == '+' || sc.ch == '-') + sc.Forward(); } else { - styler.ColourTo(sc.currentPos - 1, sc.state); - sc.ChangeState(SCE_HA_DEFAULT); + sc.SetState(SCE_HA_DEFAULT); } } - // Identifier + // Keyword or Identifier else if (sc.state == SCE_HA_IDENTIFIER) { - if (IsAWordChar(sc.ch)) { - sc.Forward(); - } else { - char s[100]; - sc.GetCurrent(s, sizeof(s)); - int style = sc.state; - int new_mode = 0; - if (keywords.InList(s)) { - style = SCE_HA_KEYWORD; - } else if (isupper(s[0])) { - if (mode >= HA_MODE_IMPORT1 && mode <= HA_MODE_IMPORT3) { - style = SCE_HA_MODULE; - new_mode = HA_MODE_IMPORT2; - } else if (mode == HA_MODE_MODULE) - style = SCE_HA_MODULE; - else - style = SCE_HA_CAPITAL; - } else if (mode == HA_MODE_IMPORT1 && - strcmp(s,"qualified") == 0) { - style = SCE_HA_KEYWORD; - new_mode = HA_MODE_IMPORT1; - } else if (mode == HA_MODE_IMPORT2) { - if (strcmp(s,"as") == 0) { - style = SCE_HA_KEYWORD; - new_mode = HA_MODE_IMPORT3; - } else if (strcmp(s,"hiding") == 0) { - style = SCE_HA_KEYWORD; - } - } else if (mode == HA_MODE_FFI) { - if (ffi.InList(s)) { - style = SCE_HA_KEYWORD; - new_mode = HA_MODE_FFI; + while (sc.More()) { + if (IsAWordChar(sc.ch, magicHash)) { + sc.Forward(); + } else if (xmode == SCE_HA_CAPITAL && sc.ch=='.') { + if (isupper(sc.chNext)) { + xmode = SCE_HA_CAPITAL; + sc.Forward(); + } else if (IsAWordStart(sc.chNext)) { + xmode = SCE_HA_IDENTIFIER; + sc.Forward(); + } else if (IsAnOperatorChar(sc.chNext)) { + xmode = SCE_HA_OPERATOR; + sc.Forward(); + } else { + break; } + } else if (xmode == SCE_HA_OPERATOR && IsAnOperatorChar(sc.ch)) { + sc.Forward(); + } else { + break; + } + } + + char s[100]; + sc.GetCurrent(s, sizeof(s)); + + int style = xmode; + + int new_mode = HA_MODE_DEFAULT; + + if (keywords.InList(s)) { + style = SCE_HA_KEYWORD; + } else if (isupper(s[0])) { + if (mode >= HA_MODE_IMPORT1 && mode <= HA_MODE_IMPORT3) { + style = SCE_HA_MODULE; + new_mode = HA_MODE_IMPORT2; + } else if (mode == HA_MODE_MODULE) { + style = SCE_HA_MODULE; } - else if (mode == HA_MODE_TYPE) { - if (strcmp(s,"family") == 0) - style = SCE_HA_KEYWORD; - } - styler.ColourTo(sc.currentPos - 1, style); - if (strcmp(s,"import") == 0 && mode != HA_MODE_FFI) - new_mode = HA_MODE_IMPORT1; - else if (strcmp(s,"module") == 0) - new_mode = HA_MODE_MODULE; - else if (strcmp(s,"foreign") == 0) + } else if (mode == HA_MODE_IMPORT1 && + strcmp(s,"qualified") == 0) { + style = SCE_HA_KEYWORD; + new_mode = HA_MODE_IMPORT1; + } else if (mode == HA_MODE_IMPORT2) { + if (strcmp(s,"as") == 0) { + style = SCE_HA_KEYWORD; + new_mode = HA_MODE_IMPORT3; + } else if (strcmp(s,"hiding") == 0) { + style = SCE_HA_KEYWORD; + } + } else if (mode == HA_MODE_TYPE) { + if (strcmp(s,"family") == 0) + style = SCE_HA_KEYWORD; + } + + if (mode == HA_MODE_FFI) { + if (ffi.InList(s)) { + style = SCE_HA_KEYWORD; new_mode = HA_MODE_FFI; - else if (strcmp(s,"type") == 0) - new_mode = HA_MODE_TYPE; - sc.ChangeState(SCE_HA_DEFAULT); - mode = new_mode; + } } + + styler.ColourTo(sc.currentPos - 1, style); + + if (strcmp(s,"import") == 0 && mode != HA_MODE_FFI) + new_mode = HA_MODE_IMPORT1; + else if (strcmp(s,"module") == 0) + new_mode = HA_MODE_MODULE; + else if (strcmp(s,"foreign") == 0) + new_mode = HA_MODE_FFI; + else if (strcmp(s,"type") == 0 + || strcmp(s,"data") == 0) + new_mode = HA_MODE_TYPE; + + xmode = 0; + sc.ChangeState(SCE_HA_DEFAULT); + mode = new_mode; }

// Comments // Oneliner else if (sc.state == SCE_HA_COMMENTLINE) { - if (sc.atLineEnd) { - styler.ColourTo(sc.currentPos - 1, sc.state); - sc.ChangeState(SCE_HA_DEFAULT); + if (xmode == 1 && sc.ch != '-') { + xmode = 0; + if (IsAnOperatorChar(sc.ch)) + sc.ChangeState(SCE_HA_OPERATOR); + } else if (sc.atLineEnd) { + sc.SetState(SCE_HA_DEFAULT); } else { sc.Forward(); } } // Nested else if (sc.state == SCE_HA_COMMENTBLOCK) { - if (sc.Match("{-")) { + if (sc.Match('{','-')) { sc.Forward(2); xmode++; } - else if (sc.Match("-}")) { + else if (sc.Match('-','}')) { sc.Forward(2); xmode--; if (xmode == 0) { - styler.ColourTo(sc.currentPos - 1, sc.state); - sc.ChangeState(SCE_HA_DEFAULT); + sc.SetState(SCE_HA_DEFAULT); } } else { if (sc.atLineEnd) { - // Remember the line state for future incremental lexing - styler.SetLineState(lineCurrent, (xmode << 4) | mode); - lineCurrent++; - } + // Remember the line state for future incremental lexing + styler.SetLineState(lineCurrent, (xmode << 4) | mode); + lineCurrent++; + } + sc.Forward(); + } + } + // Pragma + else if (sc.state == SCE_HA_PRAGMA) { + if (sc.Match("#-}")) { + sc.Forward(3); + sc.SetState(SCE_HA_DEFAULT); + } else { + sc.Forward(); + } + } + // Preprocessor + else if (sc.state == SCE_HA_PREPROCESSOR) { + if (stylingWithinPreprocessor && !IsAWordStart(sc.ch)) { + sc.SetState(SCE_HA_DEFAULT); + } else if (sc.ch == '\' && !stylingWithinPreprocessor) { + sc.Forward(2); + } else if (sc.atLineEnd) { + sc.SetState(SCE_HA_DEFAULT); + } else { sc.Forward(); } } // New state? if (sc.state == SCE_HA_DEFAULT) { // Digit - if (IsADigit(sc.ch) || - (sc.ch == '.' && IsADigit(sc.chNext)) || - (sc.ch == '-' && IsADigit(sc.chNext))) { - styler.ColourTo(sc.currentPos - 1, sc.state); - sc.ChangeState(SCE_HA_NUMBER); + if (IsADigit(sc.ch)) { + sc.SetState(SCE_HA_NUMBER); if (sc.ch == '0' && (sc.chNext == 'X' || sc.chNext == 'x')) { - // Match anything starting with "0x" or "0X", too - sc.Forward(2); - xmode = 16; + // Match anything starting with "0x" or "0X", too + sc.Forward(2); + xmode = 16; } else if (sc.ch == '0' && (sc.chNext == 'O' || sc.chNext == 'o')) { - // Match anything starting with "0x" or "0X", too - sc.Forward(2); - xmode = 8; + // Match anything starting with "0x" or "0X", too + sc.Forward(2); + xmode = 8; } else { - sc.Forward(); - xmode = 10; - } + sc.Forward(); + xmode = 10; + } mode = HA_MODE_DEFAULT; } + // Pragma + else if (sc.Match("{-#")) { + sc.SetState(SCE_HA_PRAGMA); + sc.Forward(3); + } // Comment line - else if (sc.Match("--")) { - styler.ColourTo(sc.currentPos - 1, sc.state); + else if (sc.Match('-','-')) { + sc.SetState(SCE_HA_COMMENTLINE); sc.Forward(2); - sc.ChangeState(SCE_HA_COMMENTLINE); - // Comment block + xmode = 1; } - else if (sc.Match("{-")) { - styler.ColourTo(sc.currentPos - 1, sc.state); + // Comment block + else if (sc.Match('{','-')) { + sc.SetState(SCE_HA_COMMENTBLOCK); sc.Forward(2); - sc.ChangeState(SCE_HA_COMMENTBLOCK); xmode = 1; } // String else if (sc.Match('"')) { - styler.ColourTo(sc.currentPos - 1, sc.state); + sc.SetState(SCE_HA_STRING); sc.Forward(); - sc.ChangeState(SCE_HA_STRING); } // Character else if (sc.Match(''')) { - styler.ColourTo(sc.currentPos - 1, sc.state); + sc.SetState(SCE_HA_CHARACTER); sc.Forward(); - sc.ChangeState(SCE_HA_CHARACTER); } - else if (sc.ch == '(' || sc.ch == ')' || - sc.ch == '{' || sc.ch == '}' || - sc.ch == '[' || sc.ch == ']') { - styler.ColourTo(sc.currentPos - 1, sc.state); - sc.Forward(); - styler.ColourTo(sc.currentPos - 1, SCE_HA_OPERATOR); - mode = HA_MODE_DEFAULT; - } - // Operator - else if (isascii(sc.ch) && isoperator(static_cast<char>(sc.ch))) { - styler.ColourTo(sc.currentPos - 1, sc.state); + // Preprocessor + else if (sc.atLineStart && sc.ch == '#') { + mode = HA_MODE_DEFAULT; + sc.SetState(SCE_HA_PREPROCESSOR); sc.Forward(); - sc.ChangeState(SCE_HA_OPERATOR); + } + // Operator + else if (IsAnOperatorChar(sc.ch)) { mode = HA_MODE_DEFAULT; + sc.SetState(SCE_HA_OPERATOR); + } + // Braces and punctuation + else if (sc.ch == ',' || sc.ch == ';' + || sc.ch == '(' || sc.ch == ')' + || sc.ch == '[' || sc.ch == ']' + || sc.ch == '{' || sc.ch == '}') { + sc.SetState(SCE_HA_OPERATOR); + sc.Forward(); + sc.SetState(SCE_HA_DEFAULT); } - // Keyword + // Keyword or Identifier else if (IsAWordStart(sc.ch)) { - styler.ColourTo(sc.currentPos - 1, sc.state); + xmode = isupper(sc.ch) ? SCE_HA_CAPITAL : SCE_HA_IDENTIFIER; + sc.SetState(SCE_HA_IDENTIFIER); sc.Forward(); - sc.ChangeState(SCE_HA_IDENTIFIER); } else { if (sc.atLineEnd) { - // Remember the line state for future incremental lexing - styler.SetLineState(lineCurrent, (xmode << 4) | mode); - lineCurrent++; - } + // Remember the line state for future incremental lexing + styler.SetLineState(lineCurrent, (xmode << 4) | mode); + lineCurrent++; + } sc.Forward(); } }

Modified: scintilla/lexers/LexLaTeX.cxx 539 files changed, 539 insertions(+), 0 deletions(-) =================================================================== @@ -0,0 +1,539 @@ +// Scintilla source code edit control +/** @file LexLaTeX.cxx + ** Lexer for LaTeX2e. + **/ +// Copyright 1998-2001 by Neil Hodgson neilh@scintilla.org +// The License.txt file describes the conditions under which this software may be distributed. + +// Modified by G. HU in 2013. Added folding, syntax highting inside math environments, and changed some minor behaviors. + +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <stdarg.h> +#include <assert.h> +#include <ctype.h> +#include <vector> + +#include "ILexer.h" +#include "Scintilla.h" +#include "SciLexer.h" + +#include "PropSetSimple.h" +#include "WordList.h" +#include "LexAccessor.h" +#include "Accessor.h" +#include "StyleContext.h" +#include "CharacterSet.h" +#include "LexerModule.h" +#include "LexerBase.h" + +#ifdef SCI_NAMESPACE +using namespace Scintilla; +#endif + +using namespace std; + +struct latexFoldSave { + latexFoldSave() : structLev(0) { + for (int i = 0; i < 8; ++i) openBegins[i] = 0; + } + latexFoldSave(const latexFoldSave &save) : structLev(save.structLev) { + for (int i = 0; i < 8; ++i) openBegins[i] = save.openBegins[i]; + } + int openBegins[8]; + int structLev; +}; + +class LexerLaTeX : public LexerBase { +private: + vector<int> modes; + void setMode(int line, int mode) { + if (line >= static_cast<int>(modes.size())) modes.resize(line + 1, 0); + modes[line] = mode; + } + int getMode(int line) { + if (line >= 0 && line < static_cast<int>(modes.size())) return modes[line]; + return 0; + } + void truncModes(int numLines) { + if (static_cast<int>(modes.size()) > numLines * 2 + 256) + modes.resize(numLines + 128); + } + + vector<latexFoldSave> saves; + void setSave(int line, const latexFoldSave &save) { + if (line >= static_cast<int>(saves.size())) saves.resize(line + 1); + saves[line] = save; + } + void getSave(int line, latexFoldSave &save) { + if (line >= 0 && line < static_cast<int>(saves.size())) save = saves[line]; + else { + save.structLev = 0; + for (int i = 0; i < 8; ++i) save.openBegins[i] = 0; + } + } + void truncSaves(int numLines) { + if (static_cast<int>(saves.size()) > numLines * 2 + 256) + saves.resize(numLines + 128); + } +public: + static ILexer *LexerFactoryLaTeX() { + return new LexerLaTeX(); + } + void SCI_METHOD Lex(unsigned int startPos, int length, int initStyle, IDocument *pAccess); + void SCI_METHOD Fold(unsigned int startPos, int length, int initStyle, IDocument *pAccess); +}; + +static bool latexIsSpecial(int ch) { + return (ch == '#') || (ch == '$') || (ch == '%') || (ch == '&') || (ch == '_') || + (ch == '{') || (ch == '}') || (ch == ' '); +} + +static bool latexIsBlank(int ch) { + return (ch == ' ') || (ch == '\t'); +} + +static bool latexIsBlankAndNL(int ch) { + return (ch == ' ') || (ch == '\t') || (ch == '\r') || (ch == '\n'); +} + +static bool latexIsLetter(int ch) { + return isascii(ch) && isalpha(ch); +} + +static bool latexIsTagValid(int &i, int l, Accessor &styler) { + while (i < l) { + if (styler.SafeGetCharAt(i) == '{') { + while (i < l) { + i++; + if (styler.SafeGetCharAt(i) == '}') { + return true; + } else if (!latexIsLetter(styler.SafeGetCharAt(i)) && + styler.SafeGetCharAt(i)!='*') { + return false; + } + } + } else if (!latexIsBlank(styler.SafeGetCharAt(i))) { + return false; + } + i++; + } + return false; +} + +static bool latexNextNotBlankIs(int i, Accessor &styler, char needle) { + char ch; + while (i < styler.Length()) { + ch = styler.SafeGetCharAt(i); + if (!latexIsBlankAndNL(ch) && ch != '*') { + if (ch == needle) + return true; + else + return false; + } + i++; + } + return false; +} + +static bool latexLastWordIs(int start, Accessor &styler, const char *needle) { + unsigned int i = 0; + unsigned int l = static_cast<unsigned int>(strlen(needle)); + int ini = start-l+1; + char s[32]; + + while (i < l && i < 31) { + s[i] = styler.SafeGetCharAt(ini + i); + i++; + } + s[i] = '\0'; + + return (strcmp(s, needle) == 0); +} + +static bool latexLastWordIsMathEnv(int pos, Accessor &styler) { + int i, j; + char s[32]; + const char *mathEnvs[] = { "align", "alignat", "flalign", "gather", + "multiline", "displaymath", "eqnarray", "equation" }; + if (styler.SafeGetCharAt(pos) != '}') return false; + for (i = pos - 1; i >= 0; --i) { + if (styler.SafeGetCharAt(i) == '{') break; + if (pos - i >= 20) return false; + } + if (i < 0 || i == pos - 1) return false; + ++i; + for (j = 0; i + j < pos; ++j) + s[j] = styler.SafeGetCharAt(i + j); + s[j] = '\0'; + if (j == 0) return false; + if (s[j - 1] == '*') s[--j] = '\0'; + for (i = 0; i < static_cast<int>(sizeof(mathEnvs) / sizeof(const char *)); ++i) + if (strcmp(s, mathEnvs[i]) == 0) return true; + return false; +} + +static inline void latexStateReset(int &mode, int &state) { + switch (mode) { + case 1: state = SCE_L_MATH; break; + case 2: state = SCE_L_MATH2; break; + default: state = SCE_L_DEFAULT; break; + } +} + +// There are cases not handled correctly, like $abcd\textrm{what is $x+y$}z+w$. +// But I think it's already good enough. +void SCI_METHOD LexerLaTeX::Lex(unsigned int startPos, int length, int initStyle, IDocument *pAccess) { + // startPos is assumed to be the first character of a line + Accessor styler(pAccess, &props); + styler.StartAt(startPos); + int mode = getMode(styler.GetLine(startPos) - 1); + int state = initStyle; + if (state == SCE_L_ERROR || state == SCE_L_SHORTCMD || state == SCE_L_SPECIAL) // should not happen + latexStateReset(mode, state); + + char chNext = styler.SafeGetCharAt(startPos); + char chVerbatimDelim = '\0'; + styler.StartSegment(startPos); + int lengthDoc = startPos + length; + + for (int i = startPos; i < lengthDoc; i++) { + char ch = chNext; + chNext = styler.SafeGetCharAt(i + 1); + + if (styler.IsLeadByte(ch)) { + i++; + chNext = styler.SafeGetCharAt(i + 1); + continue; + } + + if (ch == '\r' || ch == '\n') + setMode(styler.GetLine(i), mode); + + switch (state) { + case SCE_L_DEFAULT : + switch (ch) { + case '\' : + styler.ColourTo(i - 1, state); + if (latexIsLetter(chNext)) { + state = SCE_L_COMMAND; + } else if (latexIsSpecial(chNext)) { + styler.ColourTo(i + 1, SCE_L_SPECIAL); + i++; + chNext = styler.SafeGetCharAt(i + 1); + } else if (chNext == '\r' || chNext == '\n') { + styler.ColourTo(i, SCE_L_ERROR); + } else { + styler.ColourTo(i + 1, SCE_L_SHORTCMD); + if (chNext == '(') { + mode = 1; + state = SCE_L_MATH; + } else if (chNext == '[') { + mode = 2; + state = SCE_L_MATH2; + } + i++; + chNext = styler.SafeGetCharAt(i + 1); + } + break; + case '$' : + styler.ColourTo(i - 1, state); + if (chNext == '$') { + styler.ColourTo(i + 1, SCE_L_SHORTCMD); + mode = 2; + state = SCE_L_MATH2; + i++; + chNext = styler.SafeGetCharAt(i + 1); + } else { + styler.ColourTo(i, SCE_L_SHORTCMD); + mode = 1; + state = SCE_L_MATH; + } + break; + case '%' : + styler.ColourTo(i - 1, state); + state = SCE_L_COMMENT; + break; + } + break; + // These 3 will never be reached. + case SCE_L_ERROR: + case SCE_L_SPECIAL: + case SCE_L_SHORTCMD: + break; + case SCE_L_COMMAND : + if (!latexIsLetter(chNext)) { + styler.ColourTo(i, state); + if (latexNextNotBlankIs(i + 1, styler, '[' )) { + state = SCE_L_CMDOPT; + } else if (latexLastWordIs(i, styler, "\begin")) { + state = SCE_L_TAG; + } else if (latexLastWordIs(i, styler, "\end")) { + state = SCE_L_TAG2; + } else if (latexLastWordIs(i, styler, "\verb") && chNext != '*' && chNext != ' ') { + chVerbatimDelim = chNext; + state = SCE_L_VERBATIM; + } else { + latexStateReset(mode, state); + } + } + break; + case SCE_L_CMDOPT : + if (ch == ']') { + styler.ColourTo(i, state); + latexStateReset(mode, state); + } + break; + case SCE_L_TAG : + if (latexIsTagValid(i, lengthDoc, styler)) { + styler.ColourTo(i, state); + latexStateReset(mode, state); + if (latexLastWordIs(i, styler, "{verbatim}")) { + state = SCE_L_VERBATIM; + } else if (latexLastWordIs(i, styler, "{comment}")) { + state = SCE_L_COMMENT2; + } else if (latexLastWordIs(i, styler, "{math}") && mode == 0) { + mode = 1; + state = SCE_L_MATH; + } else if (latexLastWordIsMathEnv(i, styler) && mode == 0) { + mode = 2; + state = SCE_L_MATH2; + } + } else { + styler.ColourTo(i, SCE_L_ERROR); + latexStateReset(mode, state); + ch = styler.SafeGetCharAt(i); + if (ch == '\r' || ch == '\n') setMode(styler.GetLine(i), mode); + } + chNext = styler.SafeGetCharAt(i+1); + break; + case SCE_L_TAG2 : + if (latexIsTagValid(i, lengthDoc, styler)) { + styler.ColourTo(i, state); + latexStateReset(mode, state); + } else { + styler.ColourTo(i, SCE_L_ERROR); + latexStateReset(mode, state); + ch = styler.SafeGetCharAt(i); + if (ch == '\r' || ch == '\n') setMode(styler.GetLine(i), mode); + } + chNext = styler.SafeGetCharAt(i+1); + break; + case SCE_L_MATH : + switch (ch) { + case '\' : + styler.ColourTo(i - 1, state); + if (latexIsLetter(chNext)) { + int match = i + 3; + if (latexLastWordIs(match, styler, "\end")) { + match++; + if (latexIsTagValid(match, lengthDoc, styler)) { + if (latexLastWordIs(match, styler, "{math}")) + mode = 0; + } + } + state = SCE_L_COMMAND; + } else if (latexIsSpecial(chNext)) { + styler.ColourTo(i + 1, SCE_L_SPECIAL); + i++; + chNext = styler.SafeGetCharAt(i + 1); + } else if (chNext == '\r' || chNext == '\n') { + styler.ColourTo(i, SCE_L_ERROR); + } else { + if (chNext == ')') { + mode = 0; + state = SCE_L_DEFAULT; + } + styler.ColourTo(i + 1, SCE_L_SHORTCMD); + i++; + chNext = styler.SafeGetCharAt(i + 1); + } + break; + case '$' : + styler.ColourTo(i - 1, state); + styler.ColourTo(i, SCE_L_SHORTCMD); + mode = 0; + state = SCE_L_DEFAULT; + break; + case '%' : + styler.ColourTo(i - 1, state); + state = SCE_L_COMMENT; + break; + } + break; + case SCE_L_MATH2 : + switch (ch) { + case '\' : + styler.ColourTo(i - 1, state); + if (latexIsLetter(chNext)) { + int match = i + 3; + if (latexLastWordIs(match, styler, "\end")) { + match++; + if (latexIsTagValid(match, lengthDoc, styler)) { + if (latexLastWordIsMathEnv(match, styler)) + mode = 0; + } + } + state = SCE_L_COMMAND; + } else if (latexIsSpecial(chNext)) { + styler.ColourTo(i + 1, SCE_L_SPECIAL); + i++; + chNext = styler.SafeGetCharAt(i + 1); + } else if (chNext == '\r' || chNext == '\n') { + styler.ColourTo(i, SCE_L_ERROR); + } else { + if (chNext == ']') { + mode = 0; + state = SCE_L_DEFAULT; + } + styler.ColourTo(i + 1, SCE_L_SHORTCMD); + i++; + chNext = styler.SafeGetCharAt(i + 1); + } + break; + case '$' : + styler.ColourTo(i - 1, state); + if (chNext == '$') { + styler.ColourTo(i + 1, SCE_L_SHORTCMD); + i++; + chNext = styler.SafeGetCharAt(i + 1); + mode = 0; + state = SCE_L_DEFAULT; + } else { // This may not be an error, e.g. \begin{equation}\text{$a$}\end{equation} + styler.ColourTo(i, SCE_L_SHORTCMD); + } + break; + case '%' : + styler.ColourTo(i - 1, state); + state = SCE_L_COMMENT; + break; + } + break; + case SCE_L_COMMENT : + if (ch == '\r' || ch == '\n') { + styler.ColourTo(i - 1, state); + latexStateReset(mode, state); + } + break; + case SCE_L_COMMENT2 : + if (ch == '\') { + int match = i + 3; + if (latexLastWordIs(match, styler, "\end")) { + match++; + if (latexIsTagValid(match, lengthDoc, styler)) { + if (latexLastWordIs(match, styler, "{comment}")) { + styler.ColourTo(i - 1, state); + state = SCE_L_COMMAND; + } + } + } + } + break; + case SCE_L_VERBATIM : + if (ch == '\') { + int match = i + 3; + if (latexLastWordIs(match, styler, "\end")) { + match++; + if (latexIsTagValid(match, lengthDoc, styler)) { + if (latexLastWordIs(match, styler, "{verbatim}")) { + styler.ColourTo(i - 1, state); + state = SCE_L_COMMAND; + } + } + } + } else if (chNext == chVerbatimDelim) { + styler.ColourTo(i + 1, state); + latexStateReset(mode, state); + chVerbatimDelim = '\0'; + i++; + chNext = styler.SafeGetCharAt(i + 1); + } else if (chVerbatimDelim != '\0' && (ch == '\n' || ch == '\r')) { + styler.ColourTo(i, SCE_L_ERROR); + latexStateReset(mode, state); + chVerbatimDelim = '\0'; + } + break; + } + } + if (lengthDoc == styler.Length()) truncModes(styler.GetLine(lengthDoc - 1)); + styler.ColourTo(lengthDoc - 1, state); + styler.Flush(); +} + +static int latexFoldSaveToInt(const latexFoldSave &save) { + int sum = 0; + for (int i = 0; i <= save.structLev; ++i) + sum += save.openBegins[i]; + return ((sum + save.structLev + SC_FOLDLEVELBASE) & SC_FOLDLEVELNUMBERMASK); +} + +// Change folding state while processing a line +// Return the level before the first relevant command +void SCI_METHOD LexerLaTeX::Fold(unsigned int startPos, int length, int, IDocument *pAccess) { + const char *structWords[7] = {"part", "chapter", "section", "subsection", + "subsubsection", "paragraph", "subparagraph"}; + Accessor styler(pAccess, &props); + unsigned int endPos = startPos + length; + int curLine = styler.GetLine(startPos); + latexFoldSave save; + getSave(curLine - 1, save); + do { + char ch, buf[16]; + int i, j, lev = -1; + bool needFold = false; + for (i = static_cast<int>(startPos); i < static_cast<int>(endPos); ++i) { + ch = styler.SafeGetCharAt(i); + if (ch == '\r' || ch == '\n') break; + if (ch != '\' || styler.StyleAt(i) != SCE_L_COMMAND) continue; + for (j = 0; j < 15 && i + 1 < static_cast<int>(endPos); ++j, ++i) { + buf[j] = styler.SafeGetCharAt(i + 1); + if (!latexIsLetter(buf[j])) break; + } + buf[j] = '\0'; + if (strcmp(buf, "begin") == 0) { + if (lev < 0) lev = latexFoldSaveToInt(save); + ++save.openBegins[save.structLev]; + needFold = true; + } + else if (strcmp(buf, "end") == 0) { + while (save.structLev > 0 && save.openBegins[save.structLev] == 0) + --save.structLev; + if (lev < 0) lev = latexFoldSaveToInt(save); + if (save.openBegins[save.structLev] > 0) --save.openBegins[save.structLev]; + } + else { + for (j = 0; j < 7; ++j) + if (strcmp(buf, structWords[j]) == 0) break; + if (j >= 7) continue; + save.structLev = j; // level before the command + for (j = save.structLev + 1; j < 8; ++j) { + save.openBegins[save.structLev] += save.openBegins[j]; + save.openBegins[j] = 0; + } + if (lev < 0) lev = latexFoldSaveToInt(save); + ++save.structLev; // level after the command + needFold = true; + } + } + if (lev < 0) lev = latexFoldSaveToInt(save); + if (needFold) lev |= SC_FOLDLEVELHEADERFLAG; + styler.SetLevel(curLine, lev); + setSave(curLine, save); + ++curLine; + startPos = styler.LineStart(curLine); + if (static_cast<int>(startPos) == styler.Length()) { + lev = latexFoldSaveToInt(save); + styler.SetLevel(curLine, lev); + setSave(curLine, save); + truncSaves(curLine); + } + } while (startPos < endPos); + styler.Flush(); +} + +static const char *const emptyWordListDesc[] = { + 0 +}; + +LexerModule lmLatex(SCLEX_LATEX, LexerLaTeX::LexerFactoryLaTeX, "latex", emptyWordListDesc);

Modified: scintilla/lexers/LexMatlab.cxx 87 files changed, 77 insertions(+), 10 deletions(-) =================================================================== @@ -6,6 +6,12 @@ ** Changes by Christoph Dalitz 2003/12/04: ** - added support for Octave ** - Strings can now be included both in single or double quotes + ** + ** Changes by John Donoghue 2012/04/02 + ** - added block comment (and nested block comments) + ** - added ... displayed as a comment + ** - removed unused IsAWord functions + ** - added some comments **/ // Copyright 1998-2001 by Neil Hodgson neilh@scintilla.org // The License.txt file describes the conditions under which this software may be distributed. @@ -48,14 +54,6 @@ static bool IsOctaveComment(Accessor &styler, int pos, int len) { return len > 0 && IsOctaveCommentChar(styler[pos]) ; }

-static inline bool IsAWordChar(const int ch) { - return (ch < 0x80) && (isalnum(ch) || ch == '_'); -} - -static inline bool IsAWordStart(const int ch) { - return (ch < 0x80) && (isalnum(ch) || ch == '_'); -} - static void ColouriseMatlabOctaveDoc( unsigned int startPos, int length, int initStyle, WordList *keywordlists[], Accessor &styler, @@ -65,12 +63,41 @@ static void ColouriseMatlabOctaveDoc(

styler.StartAt(startPos);

+ // boolean for when the ' is allowed to be transpose vs the start/end + // of a string bool transpose = false;

+ // approximate position of first non space character in a line + int nonSpaceColumn = -1; + // approximate column position of the current character in a line + int column = 0; + + // use the line state of each line to store the block comment depth + int curLine = styler.GetLine(startPos); + int commentDepth = curLine > 0 ? styler.GetLineState(curLine-1) : 0; + + StyleContext sc(startPos, length, initStyle, styler);

- for (; sc.More(); sc.Forward()) { + for (; sc.More(); sc.Forward(), column++) { + + if(sc.atLineStart) { + // set the line state to the current commentDepth + curLine = styler.GetLine(sc.currentPos); + styler.SetLineState(curLine, commentDepth);

+ // reset the column to 0, nonSpace to -1 (not set) + column = 0; + nonSpaceColumn = -1; + } + + // save the column position of first non space character in a line + if((nonSpaceColumn == -1) && (! IsASpace(sc.ch))) + { + nonSpaceColumn = column; + } + + // check for end of states if (sc.state == SCE_MATLAB_OPERATOR) { if (sc.chPrev == '.') { if (sc.ch == '*' || sc.ch == '/' || sc.ch == '\' || sc.ch == '^') { @@ -79,6 +106,10 @@ static void ColouriseMatlabOctaveDoc( } else if (sc.ch == ''') { sc.ForwardSetState(SCE_MATLAB_DEFAULT); transpose = true; + } else if(sc.ch == '.' && sc.chNext == '.') { + // we werent an operator, but a '...' + sc.ChangeState(SCE_MATLAB_COMMENT); + transpose = false; } else { sc.SetState(SCE_MATLAB_DEFAULT); } @@ -121,15 +152,51 @@ static void ColouriseMatlabOctaveDoc( } else if (sc.ch == '"') { sc.ForwardSetState(SCE_MATLAB_DEFAULT); } - } else if (sc.state == SCE_MATLAB_COMMENT || sc.state == SCE_MATLAB_COMMAND) { + } else if (sc.state == SCE_MATLAB_COMMAND) { if (sc.atLineEnd) { sc.SetState(SCE_MATLAB_DEFAULT); transpose = false; } + } else if (sc.state == SCE_MATLAB_COMMENT) { + // end or start of a nested a block comment? + if( IsCommentChar(sc.ch) && sc.chNext == '}' && nonSpaceColumn == column) { + if(commentDepth > 0) commentDepth --; + + curLine = styler.GetLine(sc.currentPos); + styler.SetLineState(curLine, commentDepth); + sc.Forward(); + + if (commentDepth == 0) { + sc.ForwardSetState(SCE_D_DEFAULT); + transpose = false; + } + } + else if( IsCommentChar(sc.ch) && sc.chNext == '{' && nonSpaceColumn == column) + { + commentDepth ++; + + curLine = styler.GetLine(sc.currentPos); + styler.SetLineState(curLine, commentDepth); + sc.Forward(); + transpose = false; + + } else if(commentDepth == 0) { + // single line comment + if (sc.atLineEnd || sc.ch == '\r' || sc.ch == '\n') { + sc.SetState(SCE_MATLAB_DEFAULT); + transpose = false; + } + } }

+ // check start of a new state if (sc.state == SCE_MATLAB_DEFAULT) { if (IsCommentChar(sc.ch)) { + // ncrement depth if we are a block comment + if(sc.chNext == '{' && nonSpaceColumn == column) + commentDepth ++; + curLine = styler.GetLine(sc.currentPos); + styler.SetLineState(curLine, commentDepth); sc.SetState(SCE_MATLAB_COMMENT); } else if (sc.ch == '!' && sc.chNext != '=' ) { sc.SetState(SCE_MATLAB_COMMAND);

Modified: scintilla/lexers/LexOthers.cxx 301 files changed, 4 insertions(+), 297 deletions(-) =================================================================== @@ -1,7 +1,6 @@ // Scintilla source code edit control /** @file LexOthers.cxx ** Lexers for batch files, diff results, properties files, make files and error lists. - ** Also lexer for LaTeX documents. **/ // Copyright 1998-2001 by Neil Hodgson neilh@scintilla.org // The License.txt file describes the conditions under which this software may be distributed. @@ -943,6 +942,10 @@ static int RecogniseErrorListLine(const char *lineBuffer, unsigned int lengthLin strstr(lineBuffer, ".java:")) { // Java stack back trace return SCE_ERR_JAVA_STACK; + } else if (strstart(lineBuffer, "In file included from ") || + strstart(lineBuffer, " from ")) { + // GCC showing include path to following error + return SCE_ERR_GCC_INCLUDED_FROM; } else { // Look for one of the following formats: // GCC: <filename>:<line>:<message> @@ -1109,301 +1112,6 @@ static void ColouriseErrorListDoc(unsigned int startPos, int length, int, WordLi } }

-static bool latexIsSpecial(int ch) { - return (ch == '#') || (ch == '$') || (ch == '%') || (ch == '&') || (ch == '_') || - (ch == '{') || (ch == '}') || (ch == ' '); -} - -static bool latexIsBlank(int ch) { - return (ch == ' ') || (ch == '\t'); -} - -static bool latexIsBlankAndNL(int ch) { - return (ch == ' ') || (ch == '\t') || (ch == '\r') || (ch == '\n'); -} - -static bool latexIsLetter(int ch) { - return isascii(ch) && isalpha(ch); -} - -static bool latexIsTagValid(int &i, int l, Accessor &styler) { - while (i < l) { - if (styler.SafeGetCharAt(i) == '{') { - while (i < l) { - i++; - if (styler.SafeGetCharAt(i) == '}') { - return true; - } else if (!latexIsLetter(styler.SafeGetCharAt(i)) && - styler.SafeGetCharAt(i)!='*') { - return false; - } - } - } else if (!latexIsBlank(styler.SafeGetCharAt(i))) { - return false; - } - i++; - } - return false; -} - -static bool latexNextNotBlankIs(int i, int l, Accessor &styler, char needle) { - char ch; - while (i < l) { - ch = styler.SafeGetCharAt(i); - if (!latexIsBlankAndNL(ch) && ch != '*') { - if (ch == needle) - return true; - else - return false; - } - i++; - } - return false; -} - -static bool latexLastWordIs(int start, Accessor &styler, const char *needle) { - unsigned int i = 0; - unsigned int l = static_cast<unsigned int>(strlen(needle)); - int ini = start-l+1; - char s[32]; - - while (i < l && i < 32) { - s[i] = styler.SafeGetCharAt(ini + i); - i++; - } - s[i] = '\0'; - - return (strcmp(s, needle) == 0); -} - -static void ColouriseLatexDoc(unsigned int startPos, int length, int initStyle, - WordList *[], Accessor &styler) { - - styler.StartAt(startPos); - - int state = initStyle; - char chNext = styler.SafeGetCharAt(startPos); - styler.StartSegment(startPos); - int lengthDoc = startPos + length; - char chVerbatimDelim = '\0'; - - for (int i = startPos; i < lengthDoc; i++) { - char ch = chNext; - chNext = styler.SafeGetCharAt(i + 1); - - if (styler.IsLeadByte(ch)) { - i++; - chNext = styler.SafeGetCharAt(i + 1); - continue; - } - - switch (state) { - case SCE_L_DEFAULT : - switch (ch) { - case '\' : - styler.ColourTo(i - 1, state); - if (latexIsSpecial(chNext)) { - state = SCE_L_SPECIAL; - } else { - if (latexIsLetter(chNext)) { - state = SCE_L_COMMAND; - } else { - if (chNext == '(' || chNext == '[') { - styler.ColourTo(i-1, state); - styler.ColourTo(i+1, SCE_L_SHORTCMD); - state = SCE_L_MATH; - if (chNext == '[') - state = SCE_L_MATH2; - i++; - chNext = styler.SafeGetCharAt(i+1); - } else { - state = SCE_L_SHORTCMD; - } - } - } - break; - case '$' : - styler.ColourTo(i - 1, state); - state = SCE_L_MATH; - if (chNext == '$') { - state = SCE_L_MATH2; - i++; - chNext = styler.SafeGetCharAt(i + 1); - } - break; - case '%' : - styler.ColourTo(i - 1, state); - state = SCE_L_COMMENT; - break; - } - break; - case SCE_L_ERROR: - styler.ColourTo(i-1, state); - state = SCE_L_DEFAULT; - break; - case SCE_L_SPECIAL: - case SCE_L_SHORTCMD: - styler.ColourTo(i, state); - state = SCE_L_DEFAULT; - break; - case SCE_L_COMMAND : - if (!latexIsLetter(chNext)) { - styler.ColourTo(i, state); - state = SCE_L_DEFAULT; - if (latexNextNotBlankIs(i+1, lengthDoc, styler, '[' )) { - state = SCE_L_CMDOPT; - } else if (latexLastWordIs(i, styler, "\begin")) { - state = SCE_L_TAG; - } else if (latexLastWordIs(i, styler, "\end")) { - state = SCE_L_TAG2; - } else if (latexLastWordIs(i, styler, "\verb") && - chNext != '*' && chNext != ' ') { - chVerbatimDelim = chNext; - state = SCE_L_VERBATIM; - } - } - break; - case SCE_L_CMDOPT : - if (ch == ']') { - styler.ColourTo(i, state); - state = SCE_L_DEFAULT; - } - break; - case SCE_L_TAG : - if (latexIsTagValid(i, lengthDoc, styler)) { - styler.ColourTo(i, state); - state = SCE_L_DEFAULT; - if (latexLastWordIs(i, styler, "{verbatim}")) { - state = SCE_L_VERBATIM; - } else if (latexLastWordIs(i, styler, "{comment}")) { - state = SCE_L_COMMENT2; - } else if (latexLastWordIs(i, styler, "{math}")) { - state = SCE_L_MATH; - } else if (latexLastWordIs(i, styler, "{displaymath}")) { - state = SCE_L_MATH2; - } else if (latexLastWordIs(i, styler, "{equation}")) { - state = SCE_L_MATH2; - } - } else { - state = SCE_L_ERROR; - styler.ColourTo(i, state); - state = SCE_L_DEFAULT; - } - chNext = styler.SafeGetCharAt(i+1); - break; - case SCE_L_TAG2 : - if (latexIsTagValid(i, lengthDoc, styler)) { - styler.ColourTo(i, state); - state = SCE_L_DEFAULT; - } else { - state = SCE_L_ERROR; - } - chNext = styler.SafeGetCharAt(i+1); - break; - case SCE_L_MATH : - if (ch == '$') { - styler.ColourTo(i, state); - state = SCE_L_DEFAULT; - } else if (ch == '\' && chNext == ')') { - styler.ColourTo(i-1, state); - styler.ColourTo(i+1, SCE_L_SHORTCMD); - i++; - chNext = styler.SafeGetCharAt(i+1); - state = SCE_L_DEFAULT; - } else if (ch == '\') { - int match = i + 3; - if (latexLastWordIs(match, styler, "\end")) { - match++; - if (latexIsTagValid(match, lengthDoc, styler)) { - if (latexLastWordIs(match, styler, "{math}")) { - styler.ColourTo(i-1, state); - state = SCE_L_COMMAND; - } - } - } - } - - break; - case SCE_L_MATH2 : - if (ch == '$') { - if (chNext == '$') { - i++; - chNext = styler.SafeGetCharAt(i + 1); - styler.ColourTo(i, state); - state = SCE_L_DEFAULT; - } else { - styler.ColourTo(i, SCE_L_ERROR); - state = SCE_L_DEFAULT; - } - } else if (ch == '\' && chNext == ']') { - styler.ColourTo(i-1, state); - styler.ColourTo(i+1, SCE_L_SHORTCMD); - i++; - chNext = styler.SafeGetCharAt(i+1); - state = SCE_L_DEFAULT; - } else if (ch == '\') { - int match = i + 3; - if (latexLastWordIs(match, styler, "\end")) { - match++; - if (latexIsTagValid(match, lengthDoc, styler)) { - if (latexLastWordIs(match, styler, "{displaymath}")) { - styler.ColourTo(i-1, state); - state = SCE_L_COMMAND; - } else if (latexLastWordIs(match, styler, "{equation}")) { - styler.ColourTo(i-1, state); - state = SCE_L_COMMAND; - } - } - } - } - break; - case SCE_L_COMMENT : - if (ch == '\r' || ch == '\n') { - styler.ColourTo(i - 1, state); - state = SCE_L_DEFAULT; - } - break; - case SCE_L_COMMENT2 : - if (ch == '\') { - int match = i + 3; - if (latexLastWordIs(match, styler, "\end")) { - match++; - if (latexIsTagValid(match, lengthDoc, styler)) { - if (latexLastWordIs(match, styler, "{comment}")) { - styler.ColourTo(i-1, state); - state = SCE_L_COMMAND; - } - } - } - } - break; - case SCE_L_VERBATIM : - if (ch == '\') { - int match = i + 3; - if (latexLastWordIs(match, styler, "\end")) { - match++; - if (latexIsTagValid(match, lengthDoc, styler)) { - if (latexLastWordIs(match, styler, "{verbatim}")) { - styler.ColourTo(i-1, state); - state = SCE_L_COMMAND; - } - } - } - } else if (chNext == chVerbatimDelim) { - styler.ColourTo(i+1, state); - state = SCE_L_DEFAULT; - chVerbatimDelim = '\0'; - } else if (chVerbatimDelim != '\0' && (ch == '\n' || ch == '\r')) { - styler.ColourTo(i, SCE_L_ERROR); - state = SCE_L_DEFAULT; - chVerbatimDelim = '\0'; - } - break; - } - } - styler.ColourTo(lengthDoc-1, state); -} - static const char *const batchWordListDesc[] = { "Internal Commands", "External Commands", @@ -1429,5 +1137,4 @@ static void ColouriseNullDoc(unsigned int startPos, int length, int, WordList *[ LexerModule lmProps(SCLEX_PROPERTIES, ColourisePropsDoc, "props", FoldPropsDoc, emptyWordListDesc); LexerModule lmMake(SCLEX_MAKEFILE, ColouriseMakeDoc, "makefile", 0, emptyWordListDesc); LexerModule lmErrorList(SCLEX_ERRORLIST, ColouriseErrorListDoc, "errorlist", 0, emptyWordListDesc); -LexerModule lmLatex(SCLEX_LATEX, ColouriseLatexDoc, "latex", 0, emptyWordListDesc); LexerModule lmNull(SCLEX_NULL, ColouriseNullDoc, "null");

Modified: scintilla/lexers/LexPO.cxx 66 files changed, 65 insertions(+), 1 deletions(-) =================================================================== @@ -98,6 +98,12 @@ static void ColourisePODoc(unsigned int startPos, int length, int initStyle, Wor // forward to the first non-white character on the line bool atLineStart = sc.atLineStart; if (atLineStart) { + // reset line state if it is set to comment state so empty lines don't get + // comment line state, and the folding code folds comments separately, + // and anyway the styling don't use line state for comments + if (curLineState == SCE_PO_COMMENT) + curLineState = SCE_PO_DEFAULT; + while (sc.More() && ! sc.atLineEnd && isspacechar(sc.ch)) sc.Forward(); } @@ -142,8 +148,66 @@ static void ColourisePODoc(unsigned int startPos, int length, int initStyle, Wor sc.Complete(); }

+static int FindNextNonEmptyLineState(unsigned int startPos, Accessor &styler) { + unsigned int length = styler.Length(); + for (unsigned int i = startPos; i < length; i++) { + if (! isspacechar(styler[i])) { + return styler.GetLineState(styler.GetLine(i)); + } + } + return 0; +} + +static void FoldPODoc(unsigned int startPos, int length, int, WordList *[], Accessor &styler) { + if (! styler.GetPropertyInt("fold")) + return; + bool foldCompact = styler.GetPropertyInt("fold.compact") != 0; + bool foldComment = styler.GetPropertyInt("fold.comment") != 0; + + unsigned int endPos = startPos + length; + int curLine = styler.GetLine(startPos); + int lineState = styler.GetLineState(curLine); + int nextLineState; + int level = styler.LevelAt(curLine) & SC_FOLDLEVELNUMBERMASK; + int nextLevel; + int visible = 0; + int chNext = styler[startPos]; + + for (unsigned int i = startPos; i < endPos; i++) { + int ch = chNext; + chNext = styler.SafeGetCharAt(i+1); + + if (! isspacechar(ch)) { + visible++; + } else if ((ch == '\r' && chNext != '\n') || ch == '\n' || i+1 >= endPos) { + int lvl = level; + int nextLine = curLine + 1; + + nextLineState = styler.GetLineState(nextLine); + if ((lineState != SCE_PO_COMMENT || foldComment) && + nextLineState == lineState && + FindNextNonEmptyLineState(i, styler) == lineState) + nextLevel = SC_FOLDLEVELBASE + 1; + else + nextLevel = SC_FOLDLEVELBASE; + + if (nextLevel > level) + lvl |= SC_FOLDLEVELHEADERFLAG; + if (visible == 0 && foldCompact) + lvl |= SC_FOLDLEVELWHITEFLAG; + + styler.SetLevel(curLine, lvl); + + lineState = nextLineState; + curLine = nextLine; + level = nextLevel; + visible = 0; + } + } +} + static const char *const poWordListDesc[] = { 0 };

-LexerModule lmPO(SCLEX_PO, ColourisePODoc, "po", 0, poWordListDesc); +LexerModule lmPO(SCLEX_PO, ColourisePODoc, "po", FoldPODoc, poWordListDesc);

Modified: scintilla/lexers/LexRuby.cxx 2 files changed, 1 insertions(+), 1 deletions(-) =================================================================== @@ -119,7 +119,7 @@ static int ClassifyWordRb(unsigned int start, unsigned int end, WordList &keywor chAttr = SCE_RB_MODULE_NAME; else if (0 == strcmp(prevWord, "def")) chAttr = SCE_RB_DEFNAME; - else if (keywords.InList(s) && !followsDot(start - 1, styler)) { + else if (keywords.InList(s) && ((start == 0) || !followsDot(start - 1, styler))) { if (keywordIsAmbiguous(s) && keywordIsModifier(s, start, styler)) {

Modified: scintilla/lexlib/LexAccessor.h 39 files changed, 36 insertions(+), 3 deletions(-) =================================================================== @@ -12,6 +12,8 @@ namespace Scintilla { #endif

+enum EncodingType { enc8bit, encUnicode, encDBCS }; + class LexAccessor { private: IDocument *pAccess; @@ -25,6 +27,7 @@ class LexAccessor { int startPos; int endPos; int codePage; + enum EncodingType encodingType; int lenDoc; int mask; char styleBuf[bufferSize]; @@ -33,6 +36,7 @@ class LexAccessor { char chWhile; unsigned int startSeg; int startPosStyling; + int documentVersion;

void Fill(int position) { startPos = position - slopSize; @@ -51,9 +55,23 @@ class LexAccessor { public: LexAccessor(IDocument *pAccess_) : pAccess(pAccess_), startPos(extremePosition), endPos(0), - codePage(pAccess->CodePage()), lenDoc(pAccess->Length()), + codePage(pAccess->CodePage()), + encodingType(enc8bit), + lenDoc(pAccess->Length()), mask(127), validLen(0), chFlags(0), chWhile(0), - startSeg(0), startPosStyling(0) { + startSeg(0), startPosStyling(0), + documentVersion(pAccess->Version()) { + switch (codePage) { + case 65001: + encodingType = encUnicode; + break; + case 932: + case 936: + case 949: + case 950: + case 1361: + encodingType = encDBCS; + } } char operator[](int position) { if (position < startPos || position >= endPos) { @@ -75,7 +93,9 @@ class LexAccessor { bool IsLeadByte(char ch) { return pAccess->IsDBCSLeadByte(ch); } - + EncodingType Encoding() const { + return encodingType; + } bool Match(int pos, const char *s) { for (int i=0; *s; i++) { if (*s != SafeGetCharAt(pos+i)) @@ -93,6 +113,19 @@ class LexAccessor { int LineStart(int line) { return pAccess->LineStart(line); } + int LineEnd(int line) { + if (documentVersion >= dvLineEnd) { + return (static_cast<IDocumentWithLineEnd *>(pAccess))->LineEnd(line); + } else { + // Old interface means only '\r', '\n' and '\r\n' line ends. + int startNext = pAccess->LineStart(line+1); + char chLineEnd = SafeGetCharAt(startNext-1); + if (chLineEnd == '\n' && (SafeGetCharAt(startNext-2) == '\r')) + return startNext - 2; + else + return startNext - 1; + } + } int LevelAt(int line) { return pAccess->GetLevel(line); }

Modified: scintilla/lexlib/LexerBase.cxx 2 files changed, 1 insertions(+), 1 deletions(-) =================================================================== @@ -1,5 +1,5 @@ // Scintilla source code edit control -/** @file LexerSimple.cxx +/** @file LexerBase.cxx ** A simple lexer with no state. **/ // Copyright 1998-2010 by Neil Hodgson neilh@scintilla.org

Modified: scintilla/lexlib/LexerModule.cxx 4 files changed, 1 insertions(+), 3 deletions(-) =================================================================== @@ -74,11 +74,9 @@ int LexerModule::GetNumWordLists() const { }

const char *LexerModule::GetWordListDescription(int index) const { - static const char *emptyStr = ""; - assert(index < GetNumWordLists()); if (index >= GetNumWordLists()) { - return emptyStr; + return ""; } else { return wordListDescriptions[index]; }

Modified: scintilla/lexlib/StyleContext.h 117 files changed, 95 insertions(+), 22 deletions(-) =================================================================== @@ -19,6 +19,30 @@ static inline int MakeLowerCase(int ch) { return ch - 'A' + 'a'; }

+inline int UnicodeCodePoint(const unsigned char *us) { + if (us[0] < 0xC2) { + return us[0]; + } else if (us[0] < 0xE0) { + return ((us[0] & 0x1F) << 6) + (us[1] & 0x3F); + } else if (us[0] < 0xF0) { + return ((us[0] & 0xF) << 12) + ((us[1] & 0x3F) << 6) + (us[2] & 0x3F); + } else if (us[0] < 0xF5) { + return ((us[0] & 0x7) << 18) + ((us[1] & 0x3F) << 12) + ((us[2] & 0x3F) << 6) + (us[3] & 0x3F); + } + return us[0]; +} + +inline int BytesInUnicodeCodePoint(int codePoint) { + if (codePoint < 0x80) + return 1; + else if (codePoint < 0x800) + return 2; + else if (codePoint < 0x10000) + return 3; + else + return 4; +} + // All languages handled so far can treat all characters >= 0x80 as one class // which just continues the current token or starts an identifier if in default. // DBCS treated specially as the second character can be < 0x80 and hence @@ -26,23 +50,42 @@ static inline int MakeLowerCase(int ch) { class StyleContext { LexAccessor &styler; unsigned int endPos; + unsigned int lengthDocument; StyleContext &operator=(const StyleContext &); + void GetNextChar(unsigned int pos) { - chNext = static_cast<unsigned char>(styler.SafeGetCharAt(pos+1)); - if (styler.IsLeadByte(static_cast<char>(chNext))) { - chNext = chNext << 8; - chNext |= static_cast<unsigned char>(styler.SafeGetCharAt(pos+2)); + chNext = static_cast<unsigned char>(styler.SafeGetCharAt(pos+1, 0)); + if (styler.Encoding() == encUnicode) { + if (chNext >= 0x80) { + unsigned char bytes[4] = { static_cast<unsigned char>(chNext), 0, 0, 0 }; + for (int trail=1; trail<3; trail++) { + bytes[trail] = static_cast<unsigned char>(styler.SafeGetCharAt(pos+1+trail, 0)); + if (!((bytes[trail] >= 0x80) && (bytes[trail] < 0xc0))) { + bytes[trail] = 0; + break; + } + } + chNext = UnicodeCodePoint(bytes); + } + } else if (styler.Encoding() == encDBCS) { + if (styler.IsLeadByte(static_cast<char>(chNext))) { + chNext = chNext << 8; + chNext |= static_cast<unsigned char>(styler.SafeGetCharAt(pos+2, 0)); + } } // End of line? // Trigger on CR only (Mac style) or either on LF from CR+LF (Dos/Win) // or on LF alone (Unix). Avoid triggering two times on Dos/Win. - atLineEnd = (ch == '\r' && chNext != '\n') || - (ch == '\n') || - (currentPos >= endPos); + if (lineStartNext < styler.Length()) + atLineEnd = static_cast<int>(pos) >= (lineStartNext-1); + else // Last line + atLineEnd = static_cast<int>(pos) >= lineStartNext; }

public: unsigned int currentPos; + int currentLine; + int lineStartNext; bool atLineStart; bool atLineEnd; int state; @@ -55,6 +98,8 @@ class StyleContext { styler(styler_), endPos(startPos + length), currentPos(startPos), + currentLine(-1), + lineStartNext(-1), atLineEnd(false), state(initStyle & chMask), // Mask off all bits which aren't in the chMask. chPrev(0), @@ -62,18 +107,30 @@ class StyleContext { chNext(0) { styler.StartAt(startPos, chMask); styler.StartSegment(startPos); - atLineStart = static_cast<unsigned int>(styler.LineStart(styler.GetLine(startPos))) == startPos; + currentLine = styler.GetLine(startPos); + lineStartNext = styler.LineStart(currentLine+1); + lengthDocument = static_cast<unsigned int>(styler.Length()); + if (endPos == lengthDocument) + endPos++; + atLineStart = static_cast<unsigned int>(styler.LineStart(currentLine)) == startPos; unsigned int pos = currentPos; - ch = static_cast<unsigned char>(styler.SafeGetCharAt(pos)); - if (styler.IsLeadByte(static_cast<char>(ch))) { - pos++; - ch = ch << 8; - ch |= static_cast<unsigned char>(styler.SafeGetCharAt(pos)); + ch = static_cast<unsigned char>(styler.SafeGetCharAt(pos, 0)); + if (styler.Encoding() == encUnicode) { + // Get the current char + GetNextChar(pos-1); + ch = chNext; + pos += BytesInUnicodeCodePoint(ch) - 1; + } else if (styler.Encoding() == encDBCS) { + if (styler.IsLeadByte(static_cast<char>(ch))) { + pos++; + ch = ch << 8; + ch |= static_cast<unsigned char>(styler.SafeGetCharAt(pos, 0)); + } } GetNextChar(pos); } void Complete() { - styler.ColourTo(currentPos - 1, state); + styler.ColourTo(currentPos - ((currentPos > lengthDocument) ? 2 : 1), state); styler.Flush(); } bool More() const { @@ -82,12 +139,28 @@ class StyleContext { void Forward() { if (currentPos < endPos) { atLineStart = atLineEnd; + if (atLineStart) { + currentLine++; + lineStartNext = styler.LineStart(currentLine+1); + } chPrev = ch; - currentPos++; - if (ch >= 0x100) + if (styler.Encoding() == encUnicode) { + currentPos += BytesInUnicodeCodePoint(ch); + } else if (styler.Encoding() == encDBCS) { + currentPos++; + if (ch >= 0x100) + currentPos++; + } else { currentPos++; + } ch = chNext; - GetNextChar(currentPos + ((ch >= 0x100) ? 1 : 0)); + if (styler.Encoding() == encUnicode) { + GetNextChar(currentPos + BytesInUnicodeCodePoint(ch)-1); + } else if (styler.Encoding() == encDBCS) { + GetNextChar(currentPos + ((ch >= 0x100) ? 1 : 0)); + } else { + GetNextChar(currentPos); + } } else { atLineStart = false; chPrev = ' '; @@ -105,19 +178,19 @@ class StyleContext { state = state_; } void SetState(int state_) { - styler.ColourTo(currentPos - 1, state); + styler.ColourTo(currentPos - ((currentPos > lengthDocument) ? 2 : 1), state); state = state_; } void ForwardSetState(int state_) { Forward(); - styler.ColourTo(currentPos - 1, state); + styler.ColourTo(currentPos - ((currentPos > lengthDocument) ? 2 : 1), state); state = state_; } int LengthCurrent() { return currentPos - styler.GetStartSegment(); } int GetRelative(int n) { - return static_cast<unsigned char>(styler.SafeGetCharAt(currentPos+n)); + return static_cast<unsigned char>(styler.SafeGetCharAt(currentPos+n, 0)); } bool Match(char ch0) const { return ch == static_cast<unsigned char>(ch0); @@ -135,7 +208,7 @@ class StyleContext { return false; s++; for (int n=2; *s; n++) { - if (*s != styler.SafeGetCharAt(currentPos+n)) + if (*s != styler.SafeGetCharAt(currentPos+n, 0)) return false; s++; } @@ -150,7 +223,7 @@ class StyleContext { s++; for (int n=2; *s; n++) { if (static_cast<unsigned char>(*s) != - MakeLowerCase(static_cast<unsigned char>(styler.SafeGetCharAt(currentPos+n)))) + MakeLowerCase(static_cast<unsigned char>(styler.SafeGetCharAt(currentPos+n, 0)))) return false; s++; }

Modified: scintilla/lexlib/SubStyles.h 162 files changed, 162 insertions(+), 0 deletions(-) =================================================================== @@ -0,0 +1,162 @@ +// Scintilla source code edit control +/** @file SubStyles.h + ** Manage substyles for a lexer. + **/ +// Copyright 2012 by Neil Hodgson neilh@scintilla.org +// The License.txt file describes the conditions under which this software may be distributed. + +#ifndef SUBSTYLES_H +#define SUBSTYLES_H + +#ifdef SCI_NAMESPACE +namespace Scintilla { +#endif + +class WordClassifier { + int firstStyle; + int lenStyles; + std::map<std::string, int> wordToStyle; + +public: + + WordClassifier() : firstStyle(0), lenStyles(0) { + } + + void Allocate(int firstStyle_, int lenStyles_) { + firstStyle = firstStyle_; + lenStyles = lenStyles_; + wordToStyle.clear(); + } + + int Start() const { + return firstStyle; + } + + int Length() const { + return lenStyles; + } + + void Clear() { + firstStyle = 0; + lenStyles = 0; + wordToStyle.clear(); + } + + int ValueFor(const std::string &s) const { + std::map<std::string, int>::const_iterator it = wordToStyle.find(s); + if (it != wordToStyle.end()) + return it->second; + else + return -1; + } + + bool IncludesStyle(int style) const { + return (style >= firstStyle) && (style < (firstStyle + lenStyles)); + } + + void SetIdentifiers(int style, const char *identifiers) { + while (*identifiers) { + const char *cpSpace = identifiers; + while (*cpSpace && *cpSpace != ' ') + cpSpace++; + std::string word(identifiers, cpSpace - identifiers); + wordToStyle[word] = style; + identifiers = cpSpace; + if (*identifiers) + identifiers++; + } + } +}; + +class SubStyles { + int classifications; + const char *baseStyles; + int styleFirst; + int stylesAvailable; + int secondaryDistance; + int allocated; + std::vector<WordClassifier> classifiers; + + int BlockFromBaseStyle(int baseStyle) const { + for (int b=0; b < classifications; b++) { + if (baseStyle == baseStyles[b]) + return b; + } + return -1; + } + + int BlockFromStyle(int style) const { + int b = 0; + for (std::vector<WordClassifier>::const_iterator it=classifiers.begin(); it != classifiers.end(); ++it) { + if (it->IncludesStyle(style)) + return b; + b++; + } + return -1; + } + +public: + + SubStyles(const char *baseStyles_, int styleFirst_, int stylesAvailable_, int secondaryDistance_) : + classifications(0), + baseStyles(baseStyles_), + styleFirst(styleFirst_), + stylesAvailable(stylesAvailable_), + secondaryDistance(secondaryDistance_), + allocated(0) { + while (baseStyles[classifications]) { + classifications++; + classifiers.push_back(WordClassifier()); + } + } + + int Allocate(int styleBase, int numberStyles) { + int block = BlockFromBaseStyle(styleBase); + if (block >= 0) { + if ((allocated + numberStyles) > stylesAvailable) + return -1; + int startBlock = styleFirst + allocated; + allocated += numberStyles; + classifiers[block].Allocate(startBlock, numberStyles); + return startBlock; + } else { + return -1; + } + } + + int Start(int styleBase) { + int block = BlockFromBaseStyle(styleBase); + return (block >= 0) ? classifiers[block].Start() : -1; + } + + int Length(int styleBase) { + int block = BlockFromBaseStyle(styleBase); + return (block >= 0) ? classifiers[block].Length() : 0; + } + + int DistanceToSecondaryStyles() const { + return secondaryDistance; + } + + void SetIdentifiers(int style, const char *identifiers) { + int block = BlockFromStyle(style); + if (block >= 0) + classifiers[block].SetIdentifiers(style, identifiers); + } + + void Free() { + allocated = 0; + for (std::vector<WordClassifier>::iterator it=classifiers.begin(); it != classifiers.end(); ++it) + it->Clear(); + } + + const WordClassifier &Classifier(int baseStyle) const { + return classifiers[BlockFromBaseStyle(baseStyle)]; + } +}; + +#ifdef SCI_NAMESPACE +} +#endif + +#endif

Modified: scintilla/makefile.win32 1 files changed, 1 insertions(+), 0 deletions(-) =================================================================== @@ -89,6 +89,7 @@ LexVerilog.o \ LexMarkdown.o \ LexMatlab.o \ LexD.o \ +LexLaTeX.o \ LexLisp.o \ LexLua.o \ LexHaskell.o \

Modified: scintilla/src/AutoComplete.cxx 119 files changed, 110 insertions(+), 9 deletions(-) =================================================================== @@ -10,7 +10,9 @@ #include <stdio.h> #include <assert.h>

+#include <algorithm> #include <string> +#include <vector>

#include "Platform.h"

@@ -36,7 +38,8 @@ dropRestOfWord(false), ignoreCaseBehaviour(SC_CASEINSENSITIVEBEHAVIOUR_RESPECTCASE), widthLBDefault(100), - heightLBDefault(100) { + heightLBDefault(100), + autoSort(SC_ORDER_PRESORTED) { lb = ListBox::Allocate(); stopChars[0] = '\0'; fillUpChars[0] = '\0'; @@ -101,8 +104,91 @@ char AutoComplete::GetTypesep() const { return typesep; }

+struct Sorter { + AutoComplete *ac; + const char *list; + std::vector<int> indices; + + Sorter(AutoComplete *ac_, const char *list_) : ac(ac_), list(list_) { + int i = 0; + while (list[i]) { + indices.push_back(i); // word start + while (list[i] != ac->GetTypesep() && list[i] != ac->GetSeparator() && list[i]) + ++i; + indices.push_back(i); // word end + if (list[i] == ac->GetTypesep()) { + while (list[i] != ac->GetSeparator() && list[i]) + ++i; + } + if (list[i] == ac->GetSeparator()) { + ++i; + // preserve trailing separator as blank entry + if (!list[i]) { + indices.push_back(i); + indices.push_back(i); + } + } + } + indices.push_back(i); // index of last position + } + + bool operator()(int a, int b) { + int lenA = indices[a * 2 + 1] - indices[a * 2]; + int lenB = indices[b * 2 + 1] - indices[b * 2]; + int len = std::min(lenA, lenB); + int cmp; + if (ac->ignoreCase) + cmp = CompareNCaseInsensitive(list + indices[a * 2], list + indices[b * 2], len); + else + cmp = strncmp(list + indices[a * 2], list + indices[b * 2], len); + if (cmp == 0) + cmp = lenA - lenB; + return cmp < 0; + } +}; + void AutoComplete::SetList(const char *list) { - lb->SetList(lis@@ Diff output truncated at 100000 characters. @@

-------------- This E-Mail was brought to you by github_commit_mail.py (Source: https://github.com/geany/infrastructure).