[geany/geany] ecf98b: Update Scintilla to version 3.10.2

Branch: refs/heads/master Author: Colomban Wendling ban@herbesfolles.org Committer: Colomban Wendling ban@herbesfolles.org Date: Sat, 12 Jan 2019 22:52:53 UTC Commit: ecf98b1efbf9f58d808a0fe8ea1257bb10eddc93 https://github.com/geany/geany/commit/ecf98b1efbf9f58d808a0fe8ea1257bb10eddc...

Log Message: ----------- Update Scintilla to version 3.10.2

Closes #971, closes #1947.

Modified Paths: -------------- scintilla/gtk/ScintillaGTK.cxx scintilla/gtk/ScintillaGTKAccessible.cxx scintilla/include/Platform.h scintilla/include/SciLexer.h scintilla/include/Scintilla.h scintilla/include/Scintilla.iface scintilla/lexers/LexBash.cxx scintilla/lexers/LexCPP.cxx scintilla/lexers/LexFortran.cxx scintilla/lexers/LexMarkdown.cxx scintilla/lexers/LexVerilog.cxx scintilla/lexlib/CharacterCategory.cxx scintilla/lexlib/PropSetSimple.cxx scintilla/scintilla_changes.patch scintilla/src/CaseConvert.cxx scintilla/src/Catalogue.cxx scintilla/src/CellBuffer.cxx scintilla/src/CellBuffer.h scintilla/src/ContractionState.cxx scintilla/src/Document.cxx scintilla/src/Document.h scintilla/src/EditModel.cxx scintilla/src/EditView.cxx scintilla/src/Editor.cxx scintilla/src/Editor.h scintilla/src/ExternalLexer.cxx scintilla/src/Indicator.cxx scintilla/src/KeyMap.cxx scintilla/src/KeyMap.h scintilla/src/PerLine.cxx scintilla/src/PositionCache.cxx scintilla/src/PositionCache.h scintilla/src/RESearch.cxx scintilla/src/RunStyles.cxx scintilla/src/ScintillaBase.cxx scintilla/src/Style.cxx scintilla/src/UniConversion.cxx scintilla/src/UniConversion.h scintilla/src/ViewStyle.cxx scintilla/version.txt

Modified: scintilla/gtk/ScintillaGTK.cxx 7 lines changed, 4 insertions(+), 3 deletions(-) =================================================================== @@ -1075,9 +1075,10 @@ void ScintillaGTK::NotifyChange() { }

void ScintillaGTK::NotifyFocus(bool focus) { - g_signal_emit(G_OBJECT(sci), scintilla_signals[COMMAND_SIGNAL], 0, - Platform::LongFromTwoShorts - (GetCtrlID(), focus ? SCEN_SETFOCUS : SCEN_KILLFOCUS), PWidget(wMain)); + if (commandEvents) + g_signal_emit(G_OBJECT(sci), scintilla_signals[COMMAND_SIGNAL], 0, + Platform::LongFromTwoShorts + (GetCtrlID(), focus ? SCEN_SETFOCUS : SCEN_KILLFOCUS), PWidget(wMain)); Editor::NotifyFocus(focus); }

Modified: scintilla/gtk/ScintillaGTKAccessible.cxx 2 lines changed, 1 insertions(+), 1 deletions(-) =================================================================== @@ -1128,7 +1128,7 @@ AtkObject *ScintillaGTKAccessible::WidgetGetAccessibleImpl(GtkWidget *widget, At if (parent_atk_type == 0) { AtkObject *parent_obj = GTK_WIDGET_CLASS(widget_parent_class)->get_accessible(widget); if (parent_obj) { - GType parent_atk_type = G_OBJECT_TYPE(parent_obj); + parent_atk_type = G_OBJECT_TYPE(parent_obj);

// Figure out whether accessibility is enabled by looking at the type of the accessible // object which would be created for the parent type of ScintillaObject.

Modified: scintilla/include/Platform.h 5 lines changed, 5 insertions(+), 0 deletions(-) =================================================================== @@ -25,6 +25,7 @@ #define PLAT_FOX 0 #define PLAT_CURSES 0 #define PLAT_TK 0 +#define PLAT_HAIKU 0

#if defined(FOX) #undef PLAT_FOX @@ -38,6 +39,10 @@ #undef PLAT_CURSES #define PLAT_CURSES 1

+#elif defined(__HAIKU__) +#undef PLAT_HAIKU +#define PLAT_HAIKU 1 + #elif defined(SCINTILLA_QT) #undef PLAT_QT #define PLAT_QT 1

Modified: scintilla/include/SciLexer.h 48 lines changed, 48 insertions(+), 0 deletions(-) =================================================================== @@ -136,6 +136,9 @@ #define SCLEX_EDIFACT 121 #define SCLEX_INDENT 122 #define SCLEX_MAXIMA 123 +#define SCLEX_STATA 124 +#define SCLEX_SAS 125 +#define SCLEX_NIM 126 #define SCLEX_LPEG 999 #define SCLEX_AUTOMATIC 1000 #define SCE_P_DEFAULT 0 @@ -1826,6 +1829,51 @@ #define SCE_EDI_UNA 6 #define SCE_EDI_UNH 7 #define SCE_EDI_BADSEGMENT 8 +#define SCE_STATA_DEFAULT 0 +#define SCE_STATA_COMMENT 1 +#define SCE_STATA_COMMENTLINE 2 +#define SCE_STATA_COMMENTBLOCK 3 +#define SCE_STATA_NUMBER 4 +#define SCE_STATA_OPERATOR 5 +#define SCE_STATA_IDENTIFIER 6 +#define SCE_STATA_STRING 7 +#define SCE_STATA_TYPE 8 +#define SCE_STATA_WORD 9 +#define SCE_STATA_GLOBAL_MACRO 10 +#define SCE_STATA_MACRO 11 +#define SCE_SAS_DEFAULT 0 +#define SCE_SAS_COMMENT 1 +#define SCE_SAS_COMMENTLINE 2 +#define SCE_SAS_COMMENTBLOCK 3 +#define SCE_SAS_NUMBER 4 +#define SCE_SAS_OPERATOR 5 +#define SCE_SAS_IDENTIFIER 6 +#define SCE_SAS_STRING 7 +#define SCE_SAS_TYPE 8 +#define SCE_SAS_WORD 9 +#define SCE_SAS_GLOBAL_MACRO 10 +#define SCE_SAS_MACRO 11 +#define SCE_SAS_MACRO_KEYWORD 12 +#define SCE_SAS_BLOCK_KEYWORD 13 +#define SCE_SAS_MACRO_FUNCTION 14 +#define SCE_SAS_STATEMENT 15 +#define SCE_NIM_DEFAULT 0 +#define SCE_NIM_COMMENT 1 +#define SCE_NIM_COMMENTDOC 2 +#define SCE_NIM_COMMENTLINE 3 +#define SCE_NIM_COMMENTLINEDOC 4 +#define SCE_NIM_NUMBER 5 +#define SCE_NIM_STRING 6 +#define SCE_NIM_CHARACTER 7 +#define SCE_NIM_WORD 8 +#define SCE_NIM_TRIPLE 9 +#define SCE_NIM_TRIPLEDOUBLE 10 +#define SCE_NIM_BACKTICKS 11 +#define SCE_NIM_FUNCNAME 12 +#define SCE_NIM_STRINGEOL 13 +#define SCE_NIM_NUMERROR 14 +#define SCE_NIM_OPERATOR 15 +#define SCE_NIM_IDENTIFIER 16 /* --Autogenerated -- end of section automatically generated from Scintilla.iface */

#endif

Modified: scintilla/include/Scintilla.h 14 lines changed, 14 insertions(+), 0 deletions(-) =================================================================== @@ -365,6 +365,7 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam, #define SCI_GETLINEINDENTPOSITION 2128 #define SCI_GETCOLUMN 2129 #define SCI_COUNTCHARACTERS 2633 +#define SCI_COUNTCODEUNITS 2715 #define SCI_SETHSCROLLBAR 2130 #define SCI_GETHSCROLLBAR 2131 #define SC_IV_NONE 0 @@ -698,6 +699,8 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam, #define SCI_RELEASEDOCUMENT 2377 #define SCI_GETDOCUMENTOPTIONS 2379 #define SCI_GETMODEVENTMASK 2378 +#define SCI_SETCOMMANDEVENTS 2717 +#define SCI_GETCOMMANDEVENTS 2718 #define SCI_SETFOCUS 2380 #define SCI_GETFOCUS 2381 #define SC_STATUS_OK 0 @@ -755,6 +758,7 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam, #define SCI_POSITIONBEFORE 2417 #define SCI_POSITIONAFTER 2418 #define SCI_POSITIONRELATIVE 2670 +#define SCI_POSITIONRELATIVECODEUNITS 2716 #define SCI_COPYRANGE 2419 #define SCI_COPYTEXT 2420 #define SC_SEL_STREAM 0 @@ -1108,6 +1112,16 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam, #define SCN_AUTOCCOMPLETED 2030 #define SCN_MARGINRIGHTCLICK 2031 #define SCN_AUTOCSELECTIONCHANGE 2032 +#ifndef SCI_DISABLE_PROVISIONAL +#define SC_LINECHARACTERINDEX_NONE 0 +#define SC_LINECHARACTERINDEX_UTF32 1 +#define SC_LINECHARACTERINDEX_UTF16 2 +#define SCI_GETLINECHARACTERINDEX 2710 +#define SCI_ALLOCATELINECHARACTERINDEX 2711 +#define SCI_RELEASELINECHARACTERINDEX 2712 +#define SCI_LINEFROMINDEXPOSITION 2713 +#define SCI_INDEXPOSITIONFROMLINE 2714 +#endif /* --Autogenerated -- end of section automatically generated from Scintilla.iface */

/* These structures are defined to be exactly the same shape as the Win32

Modified: scintilla/include/Scintilla.iface 90 lines changed, 88 insertions(+), 2 deletions(-) =================================================================== @@ -862,6 +862,9 @@ get int GetColumn=2129(position pos,) # Count characters between two positions. fun int CountCharacters=2633(position start, position end)

+# Count code units between two positions. +fun int CountCodeUnits=2715(position start, position end) + # Show or hide the horizontal scroll bar. set void SetHScrollBar=2130(bool visible,) # Is the horizontal scroll bar visible? @@ -1803,6 +1806,12 @@ get int GetDocumentOptions=2379(,) # Get which document modification events are sent to the container. get int GetModEventMask=2378(,)

+# Set whether command events are sent to the container. +set void SetCommandEvents=2717(bool commandEvents,) + +# Get whether command events are sent to the container. +get bool GetCommandEvents=2718(,) + # Change internal focus flag. set void SetFocus=2380(bool focus,) # Get internal focus flag. @@ -1966,6 +1975,11 @@ fun position PositionAfter=2418(position pos,) # of characters. Returned value is always between 0 and last position in document. fun position PositionRelative=2670(position pos, int relative)

+# Given a valid document position, return a position that differs in a number +# of UTF-16 code units. Returned value is always between 0 and last position in document. +# The result may point half way (2 bytes) inside a non-BMP character. +fun position PositionRelativeCodeUnits=2716(position pos, int relative) + # Copy a range of text to the clipboard. Positions are clipped into the document. fun void CopyRange=2419(position start, position end)

@@ -2939,6 +2953,9 @@ val SCLEX_JSON=120 val SCLEX_EDIFACT=121 val SCLEX_INDENT=122 val SCLEX_MAXIMA=123 +val SCLEX_STATA=124 +val SCLEX_SAS=125 +val SCLEX_NIM=126 val SCLEX_LPEG=999

# When a lexer specifies its language as SCLEX_AUTOMATIC it receives a @@ -4861,6 +4878,57 @@ val SCE_EDI_SEP_RELEASE=5 val SCE_EDI_UNA=6 val SCE_EDI_UNH=7 val SCE_EDI_BADSEGMENT=8 +# Lexical states for SCLEX_STATA +lex STATA=SCLEX_STATA SCE_STATA_ +val SCE_STATA_DEFAULT=0 +val SCE_STATA_COMMENT=1 +val SCE_STATA_COMMENTLINE=2 +val SCE_STATA_COMMENTBLOCK=3 +val SCE_STATA_NUMBER=4 +val SCE_STATA_OPERATOR=5 +val SCE_STATA_IDENTIFIER=6 +val SCE_STATA_STRING=7 +val SCE_STATA_TYPE=8 +val SCE_STATA_WORD=9 +val SCE_STATA_GLOBAL_MACRO=10 +val SCE_STATA_MACRO=11 +# Lexical states for SCLEX_SAS +lex SAS=SCLEX_SAS SCE_SAS_ +val SCE_SAS_DEFAULT=0 +val SCE_SAS_COMMENT=1 +val SCE_SAS_COMMENTLINE=2 +val SCE_SAS_COMMENTBLOCK=3 +val SCE_SAS_NUMBER=4 +val SCE_SAS_OPERATOR=5 +val SCE_SAS_IDENTIFIER=6 +val SCE_SAS_STRING=7 +val SCE_SAS_TYPE=8 +val SCE_SAS_WORD=9 +val SCE_SAS_GLOBAL_MACRO=10 +val SCE_SAS_MACRO=11 +val SCE_SAS_MACRO_KEYWORD=12 +val SCE_SAS_BLOCK_KEYWORD=13 +val SCE_SAS_MACRO_FUNCTION=14 +val SCE_SAS_STATEMENT=15 +# Lexical states for SCLEX_NIM +lex Nim=SCLEX_NIM SCE_NIM_ +val SCE_NIM_DEFAULT=0 +val SCE_NIM_COMMENT=1 +val SCE_NIM_COMMENTDOC=2 +val SCE_NIM_COMMENTLINE=3 +val SCE_NIM_COMMENTLINEDOC=4 +val SCE_NIM_NUMBER=5 +val SCE_NIM_STRING=6 +val SCE_NIM_CHARACTER=7 +val SCE_NIM_WORD=8 +val SCE_NIM_TRIPLE=9 +val SCE_NIM_TRIPLEDOUBLE=10 +val SCE_NIM_BACKTICKS=11 +val SCE_NIM_FUNCNAME=12 +val SCE_NIM_STRINGEOL=13 +val SCE_NIM_NUMERROR=14 +val SCE_NIM_OPERATOR=15 +val SCE_NIM_IDENTIFIER=16

# Events

@@ -4898,10 +4966,28 @@ evt void AutoCCompleted=2030(string text, int position, int ch, CompletionMethod evt void MarginRightClick=2031(int modifiers, int position, int margin) evt void AutoCSelectionChange=2032(int listType, string text, int position)

-# There are no provisional APIs currently. - cat Provisional

+enu LineCharacterIndexType=SC_LINECHARACTERINDEX_ +val SC_LINECHARACTERINDEX_NONE=0 +val SC_LINECHARACTERINDEX_UTF32=1 +val SC_LINECHARACTERINDEX_UTF16=2 + +# Retrieve line character index state. +get int GetLineCharacterIndex=2710(,) + +# Request line character index be created or its use count increased. +fun void AllocateLineCharacterIndex=2711(int lineCharacterIndex,) + +# Decrease use count of line character index and remove if 0. +fun void ReleaseLineCharacterIndex=2712(int lineCharacterIndex,) + +# Retrieve the document line containing a position measured in index units. +fun int LineFromIndexPosition=2713(position pos, int lineCharacterIndex) + +# Retrieve the position measured in index units at the start of a document line. +fun position IndexPositionFromLine=2714(int line, int lineCharacterIndex) + cat Deprecated

# Divide each styling byte into lexical class bits (default: 5) and indicator

Modified: scintilla/lexers/LexBash.cxx 279 lines changed, 235 insertions(+), 44 deletions(-) =================================================================== @@ -12,16 +12,23 @@ #include <stdarg.h> #include <assert.h>

+#include <string> +#include <vector> +#include <map> + #include "ILexer.h" #include "Scintilla.h" #include "SciLexer.h"

+#include "StringCopy.h" #include "WordList.h" #include "LexAccessor.h" -#include "Accessor.h" #include "StyleContext.h" #include "CharacterSet.h" #include "LexerModule.h" +#include "OptionSet.h" +#include "SubStyles.h" +#include "DefaultLexer.h"

using namespace Scintilla;

@@ -58,7 +65,9 @@ using namespace Scintilla;

#define BASH_DELIM_STACK_MAX 7

-static inline int translateBashDigit(int ch) { +namespace { + +inline int translateBashDigit(int ch) { if (ch >= '0' && ch <= '9') { return ch - '0'; } else if (ch >= 'a' && ch <= 'z') { @@ -73,7 +82,7 @@ static inline int translateBashDigit(int ch) { return BASH_BASE_ERROR; }

-static inline int getBashNumberBase(char *s) { +inline int getBashNumberBase(char *s) { int i = 0; int base = 0; while (*s) { @@ -86,15 +95,15 @@ static inline int getBashNumberBase(char *s) { return base; }

-static int opposite(int ch) { +int opposite(int ch) { if (ch == '(') return ')'; if (ch == '[') return ']'; if (ch == '{') return '}'; if (ch == '<') return '>'; return ch; }

-static int GlobScan(StyleContext &sc) { +int GlobScan(StyleContext &sc) { // forward scan for zsh globs, disambiguate versus bash arrays // complex expressions may still fail, e.g. unbalanced () '' "" etc int c, sLen = 0; @@ -120,10 +129,171 @@ static int GlobScan(StyleContext &sc) { return 0; }

-static void ColouriseBashDoc(Sci_PositionU startPos, Sci_Position length, int initStyle, - WordList *keywordlists[], Accessor &styler) { +bool IsCommentLine(Sci_Position line, LexAccessor &styler) { + Sci_Position pos = styler.LineStart(line); + Sci_Position eol_pos = styler.LineStart(line + 1) - 1; + for (Sci_Position i = pos; i < eol_pos; i++) { + char ch = styler[i]; + if (ch == '#') + return true; + else if (ch != ' ' && ch != '\t') + return false; + } + return false; +}

- WordList &keywords = *keywordlists[0]; +struct OptionsBash { + bool fold; + bool foldComment; + bool foldCompact; + + OptionsBash() { + fold = false; + foldComment = false; + foldCompact = true; + } +}; + +const char * const bashWordListDesc[] = { + "Keywords", + 0 +}; + +struct OptionSetBash : public OptionSet<OptionsBash> { + OptionSetBash() { + DefineProperty("fold", &OptionsBash::fold); + + DefineProperty("fold.comment", &OptionsBash::foldComment); + + DefineProperty("fold.compact", &OptionsBash::foldCompact); + + DefineWordListSets(bashWordListDesc); + } +}; + +const char styleSubable[] = { SCE_SH_IDENTIFIER, SCE_SH_SCALAR, 0 }; + +LexicalClass lexicalClasses[] = { + // Lexer Bash SCLEX_BASH SCE_SH_: + 0, "SCE_SH_DEFAULT", "default", "White space", + 1, "SCE_SH_ERROR", "error", "Error", + 2, "SCE_SH_COMMENTLINE", "comment line", "Line comment: #", + 3, "SCE_SH_NUMBER", "literal numeric", "Number", + 4, "SCE_SH_WORD", "keyword", "Keyword", + 5, "SCE_SH_STRING", "literal string", "String", + 6, "SCE_SH_CHARACTER", "literal string", "Single quoted string", + 7, "SCE_SH_OPERATOR", "operator", "Operators", + 8, "SCE_SH_IDENTIFIER", "identifier", "Identifiers", + 9, "SCE_SH_SCALAR", "identifier", "Scalar variable", + 10, "SCE_SH_PARAM", "identifier", "Parameter", + 11, "SCE_SH_BACKTICKS", "literal string", "Backtick quoted command", + 12, "SCE_SH_HERE_DELIM", "operator", "Heredoc delimiter", + 13, "SCE_SH_HERE_Q", "literal string", "Heredoc quoted string", +}; + +} + +class LexerBash : public DefaultLexer { + WordList keywords; + OptionsBash options; + OptionSetBash osBash; + enum { ssIdentifier, ssScalar }; + SubStyles subStyles; +public: + LexerBash() : + DefaultLexer(lexicalClasses, ELEMENTS(lexicalClasses)), + subStyles(styleSubable, 0x80, 0x40, 0) { + } + virtual ~LexerBash() { + } + void SCI_METHOD Release() override { + delete this; + } + int SCI_METHOD Version() const override { + return lvMetaData; + } + const char * SCI_METHOD PropertyNames() override { + return osBash.PropertyNames(); + } + int SCI_METHOD PropertyType(const char* name) override { + return osBash.PropertyType(name); + } + const char * SCI_METHOD DescribeProperty(const char *name) override { + return osBash.DescribeProperty(name); + } + Sci_Position SCI_METHOD PropertySet(const char *key, const char *val) override; + const char * SCI_METHOD DescribeWordListSets() override { + return osBash.DescribeWordListSets(); + } + Sci_Position SCI_METHOD WordListSet(int n, const char *wl) override; + void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override; + void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override; + + void * SCI_METHOD PrivateCall(int, void *) override { + return 0; + } + + int SCI_METHOD AllocateSubStyles(int styleBase, int numberStyles) override { + return subStyles.Allocate(styleBase, numberStyles); + } + int SCI_METHOD SubStylesStart(int styleBase) override { + return subStyles.Start(styleBase); + } + int SCI_METHOD SubStylesLength(int styleBase) override { + return subStyles.Length(styleBase); + } + int SCI_METHOD StyleFromSubStyle(int subStyle) override { + const int styleBase = subStyles.BaseStyle(subStyle); + return styleBase; + } + int SCI_METHOD PrimaryStyleFromStyle(int style) override { + return style; + } + void SCI_METHOD FreeSubStyles() override { + subStyles.Free(); + } + void SCI_METHOD SetIdentifiers(int style, const char *identifiers) override { + subStyles.SetIdentifiers(style, identifiers); + } + int SCI_METHOD DistanceToSecondaryStyles() override { + return 0; + } + const char *SCI_METHOD GetSubStyleBases() override { + return styleSubable; + } + + static ILexer *LexerFactoryBash() { + return new LexerBash(); + } +}; + +Sci_Position SCI_METHOD LexerBash::PropertySet(const char *key, const char *val) { + if (osBash.PropertySet(&options, key, val)) { + return 0; + } + return -1; +} + +Sci_Position SCI_METHOD LexerBash::WordListSet(int n, const char *wl) { + WordList *wordListN = 0; + switch (n) { + case 0: + wordListN = &keywords; + break; + } + Sci_Position firstModification = -1; + if (wordListN) { + WordList wlNew; + wlNew.Set(wl); + if (*wordListN != wlNew) { + wordListN->Set(wl); + firstModification = 0; + } + } + return firstModification; +} + +void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) { WordList cmdDelimiter, bashStruct, bashStruct_in; cmdDelimiter.Set("| || |& & && ; ;; ( ) { }"); bashStruct.Set("if elif fi while until else then do done esac eval"); @@ -237,11 +407,15 @@ static void ColouriseBashDoc(Sci_PositionU startPos, Sci_Position length, int in }; QuoteStackCls QuoteStack;

+ const WordClassifier &classifierIdentifiers = subStyles.Classifier(SCE_SH_IDENTIFIER); + const WordClassifier &classifierScalars = subStyles.Classifier(SCE_SH_SCALAR); + int numBase = 0; int digit; Sci_PositionU endPos = startPos + length; int cmdState = BASH_CMD_START; int testExprType = 0; + LexAccessor styler(pAccess);

// Always backtracks to the start of a line that is not a continuation // of the previous line (i.e. start of a bash command segment) @@ -306,6 +480,11 @@ static void ColouriseBashDoc(Sci_PositionU startPos, Sci_Position length, int in char s[500]; char s2[10]; sc.GetCurrent(s, sizeof(s)); + int identifierStyle = SCE_SH_IDENTIFIER; + int subStyle = classifierIdentifiers.ValueFor(s); + if (subStyle >= 0) { + identifierStyle = subStyle; + } // allow keywords ending in a whitespace or command delimiter s2[0] = static_cast<char>(sc.ch); s2[1] = '\0'; @@ -317,7 +496,7 @@ static void ColouriseBashDoc(Sci_PositionU startPos, Sci_Position length, int in else if (strcmp(s, "do") == 0 && keywordEnds) cmdStateNew = BASH_CMD_START; else - sc.ChangeState(SCE_SH_IDENTIFIER); + sc.ChangeState(identifierStyle); sc.SetState(SCE_SH_DEFAULT); break; } @@ -327,42 +506,49 @@ static void ColouriseBashDoc(Sci_PositionU startPos, Sci_Position length, int in cmdStateNew = BASH_CMD_TEST; testExprType = 0; } else - sc.ChangeState(SCE_SH_IDENTIFIER); + sc.ChangeState(identifierStyle); } // detect bash construct keywords else if (bashStruct.InList(s)) { if (cmdState == BASH_CMD_START && keywordEnds) cmdStateNew = BASH_CMD_START; else - sc.ChangeState(SCE_SH_IDENTIFIER); + sc.ChangeState(identifierStyle); } // 'for'|'case'|'select' needs 'in'|'do' to be highlighted later else if (bashStruct_in.InList(s)) { if (cmdState == BASH_CMD_START && keywordEnds) cmdStateNew = BASH_CMD_WORD; else - sc.ChangeState(SCE_SH_IDENTIFIER); + sc.ChangeState(identifierStyle); } // disambiguate option items and file test operators else if (s[0] == '-') { if (cmdState != BASH_CMD_TEST) - sc.ChangeState(SCE_SH_IDENTIFIER); + sc.ChangeState(identifierStyle); } // disambiguate keywords and identifiers else if (cmdState != BASH_CMD_START || !(keywords.InList(s) && keywordEnds)) { - sc.ChangeState(SCE_SH_IDENTIFIER); + sc.ChangeState(identifierStyle); } sc.SetState(SCE_SH_DEFAULT); } break; case SCE_SH_IDENTIFIER: - if (sc.chPrev == '\') { // for escaped chars - sc.ForwardSetState(SCE_SH_DEFAULT); - } else if (!setWord.Contains(sc.ch)) { - sc.SetState(SCE_SH_DEFAULT); - } else if (cmdState == BASH_CMD_ARITH && !setWordStart.Contains(sc.ch)) { - sc.SetState(SCE_SH_DEFAULT); + if (sc.chPrev == '\' || !setWord.Contains(sc.ch) || + (cmdState == BASH_CMD_ARITH && !setWordStart.Contains(sc.ch))) { + char s[500]; + sc.GetCurrent(s, sizeof(s)); + int subStyle = classifierIdentifiers.ValueFor(s); + if (subStyle >= 0) { + sc.ChangeState(subStyle); + } + if (sc.chPrev == '\') { // for escaped chars + sc.ForwardSetState(SCE_SH_DEFAULT); + } else { + sc.SetState(SCE_SH_DEFAULT); + } } break; case SCE_SH_NUMBER: @@ -516,6 +702,12 @@ static void ColouriseBashDoc(Sci_PositionU startPos, Sci_Position length, int in break; case SCE_SH_SCALAR: // variable names if (!setParam.Contains(sc.ch)) { + char s[500]; + sc.GetCurrent(s, sizeof(s)); + int subStyle = classifierScalars.ValueFor(&s[1]); // skip the $ + if (subStyle >= 0) { + sc.ChangeState(subStyle); + } if (sc.LengthCurrent() == 1) { // Special variable: $(, $_ etc. sc.ForwardSetState(SCE_SH_DEFAULT); @@ -799,23 +991,12 @@ static void ColouriseBashDoc(Sci_PositionU startPos, Sci_Position length, int in sc.Complete(); }

-static bool IsCommentLine(Sci_Position line, Accessor &styler) { - Sci_Position pos = styler.LineStart(line); - Sci_Position eol_pos = styler.LineStart(line + 1) - 1; - for (Sci_Position i = pos; i < eol_pos; i++) { - char ch = styler[i]; - if (ch == '#') - return true; - else if (ch != ' ' && ch != '\t') - return false; - } - return false; -} +void SCI_METHOD LexerBash::Fold(Sci_PositionU startPos, Sci_Position length, int, IDocument *pAccess) { + if(!options.fold) + return; + + LexAccessor styler(pAccess);

-static void FoldBashDoc(Sci_PositionU startPos, Sci_Position length, int, WordList *[], - Accessor &styler) { - bool foldComment = styler.GetPropertyInt("fold.comment") != 0; - bool foldCompact = styler.GetPropertyInt("fold.compact", 1) != 0; Sci_PositionU endPos = startPos + length; int visibleChars = 0; int skipHereCh = 0; @@ -824,14 +1005,16 @@ static void FoldBashDoc(Sci_PositionU startPos, Sci_Position length, int, WordLi int levelCurrent = levelPrev; char chNext = styler[startPos]; int styleNext = styler.StyleAt(startPos); + char word[8] = { '\0' }; // we're not interested in long words anyway + unsigned int wordlen = 0; for (Sci_PositionU i = startPos; i < endPos; i++) { char ch = chNext; chNext = styler.SafeGetCharAt(i + 1); int style = styleNext; styleNext = styler.StyleAt(i + 1); bool atEOL = (ch == '\r' && chNext != '\n') || (ch == '\n'); // Comment folding - if (foldComment && atEOL && IsCommentLine(lineCurrent, styler)) + if (options.foldComment && atEOL && IsCommentLine(lineCurrent, styler)) { if (!IsCommentLine(lineCurrent - 1, styler) && IsCommentLine(lineCurrent + 1, styler)) @@ -840,6 +1023,19 @@ static void FoldBashDoc(Sci_PositionU startPos, Sci_Position length, int, WordLi && !IsCommentLine(lineCurrent + 1, styler)) levelCurrent--; } + if (style == SCE_SH_WORD) { + if ((wordlen + 1) < sizeof(word)) + word[wordlen++] = ch; + if (styleNext != style) { + word[wordlen] = '\0'; + wordlen = 0; + if (strcmp(word, "if") == 0 || strcmp(word, "case") == 0 || strcmp(word, "do") == 0) { + levelCurrent++; + } else if (strcmp(word, "fi") == 0 || strcmp(word, "esac") == 0 || strcmp(word, "done") == 0) { + levelCurrent--; + } + } + } if (style == SCE_SH_OPERATOR) { if (ch == '{') { levelCurrent++; @@ -865,7 +1061,7 @@ static void FoldBashDoc(Sci_PositionU startPos, Sci_Position length, int, WordLi } if (atEOL) { int lev = levelPrev; - if (visibleChars == 0 && foldCompact) + if (visibleChars == 0 && options.foldCompact) lev |= SC_FOLDLEVELWHITEFLAG; if ((levelCurrent > levelPrev) && (visibleChars > 0)) lev |= SC_FOLDLEVELHEADERFLAG; @@ -884,9 +1080,4 @@ static void FoldBashDoc(Sci_PositionU startPos, Sci_Position length, int, WordLi styler.SetLevel(lineCurrent, levelPrev | flagsNext); }

-static const char * const bashWordListDesc[] = { - "Keywords", - 0 -}; - -LexerModule lmBash(SCLEX_BASH, ColouriseBashDoc, "bash", FoldBashDoc, bashWordListDesc); +LexerModule lmBash(SCLEX_BASH, LexerBash::LexerFactoryBash, "bash", bashWordListDesc);

Modified: scintilla/lexers/LexCPP.cxx 56 lines changed, 36 insertions(+), 20 deletions(-) =================================================================== @@ -201,17 +201,27 @@ struct EscapeSequence {

std::string GetRestOfLine(LexAccessor &styler, Sci_Position start, bool allowSpace) { std::string restOfLine; - Sci_Position i =0; + Sci_Position line = styler.GetLine(start); + Sci_Position pos = start; + Sci_Position endLine = styler.LineEnd(line); char ch = styler.SafeGetCharAt(start, '\n'); - const Sci_Position endLine = styler.LineEnd(styler.GetLine(start)); - while (((start+i) < endLine) && (ch != '\r')) { - const char chNext = styler.SafeGetCharAt(start + i + 1, '\n'); - if (ch == '/' && (chNext == '/' || chNext == '*')) - break; - if (allowSpace || (ch != ' ')) - restOfLine += ch; - i++; - ch = chNext; + while (pos < endLine) { + if (ch == '\' && ((pos + 1) == endLine)) { + // Continuation line + line++; + pos = styler.LineStart(line); + endLine = styler.LineEnd(line); + ch = styler.SafeGetCharAt(pos, '\n'); + } else { + const char chNext = styler.SafeGetCharAt(pos + 1, '\n'); + if (ch == '/' && (chNext == '/' || chNext == '*')) + break; + if (allowSpace || (ch != ' ')) { + restOfLine += ch; + } + pos++; + ch = chNext; + } } return restOfLine; } @@ -242,7 +252,11 @@ class LinePPState { return level >= 0 && level < 32; } int maskLevel() const noexcept { - return 1 << level; + if (level >= 0) { + return 1 << level; + } else { + return 1; + } } public: LinePPState() : state(0), ifTaken(0), level(-1) { @@ -470,7 +484,8 @@ class LexerCPP : public ILexerWithMetaData { bool caseSensitive; CharacterSet setWord; CharacterSet setNegationOp; - CharacterSet setArithmethicOp; + CharacterSet setAddOp; + CharacterSet setMultOp; CharacterSet setRelOp; CharacterSet setLogicalOp; CharacterSet setWordStart; @@ -511,7 +526,8 @@ class LexerCPP : public ILexerWithMetaData { caseSensitive(caseSensitive_), setWord(CharacterSet::setAlphaNum, "._", 0x80, true), setNegationOp(CharacterSet::setNone, "!"), - setArithmethicOp(CharacterSet::setNone, "+-/*%"), + setAddOp(CharacterSet::setNone, "+-"), + setMultOp(CharacterSet::setNone, "*/%"), setRelOp(CharacterSet::setNone, "=!<>"), setLogicalOp(CharacterSet::setNone, "|&"), subStyles(styleSubable, 0x80, 0x40, activeFlag) { @@ -1275,7 +1291,7 @@ void SCI_METHOD LexerCPP::Lex(Sci_PositionU startPos, Sci_Position length, int i // Ensure only one chosen out of #if .. #elif .. #elif .. #else .. #endif if (!preproc.CurrentIfTaken()) { // Similar to #if - std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 2, true); + std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 4, true); const bool ifGood = EvaluateExpression(restOfLine, preprocessorDefinitions); if (ifGood) { preproc.InvertCurrentLevel(); @@ -1614,13 +1630,15 @@ void LexerCPP::EvaluateTokens(std::vectorstd::string &tokens, const SymbolTabl }

// Evaluate expressions in precedence order - enum precedence { precArithmetic, precRelative, precLogical }; - for (int prec=precArithmetic; prec <= precLogical; prec++) { + enum precedence { precMult, precAdd, precRelative + , precLogical, /* end marker */ precLast }; + for (int prec = precMult; prec < precLast; prec++) { // Looking at 3 tokens at a time so end at 2 before end for (size_t k=0; (k+2)<tokens.size();) { const char chOp = tokens[k+1][0]; if ( - ((prec==precArithmetic) && setArithmethicOp.Contains(chOp)) || + ((prec==precMult) && setMultOp.Contains(chOp)) || + ((prec==precAdd) && setAddOp.Contains(chOp)) || ((prec==precRelative) && setRelOp.Contains(chOp)) || ((prec==precLogical) && setLogicalOp.Contains(chOp)) ) { @@ -1653,11 +1671,9 @@ void LexerCPP::EvaluateTokens(std::vectorstd::string &tokens, const SymbolTabl result = valA || valB; else if (tokens[k+1] == "&&") result = valA && valB; - char sResult[30]; - sprintf(sResult, "%d", result); std::vectorstd::string::iterator itInsert = tokens.erase(tokens.begin() + k, tokens.begin() + k + 3); - tokens.insert(itInsert, sResult); + tokens.insert(itInsert, std::to_string(result)); } else { k++; }

Modified: scintilla/lexers/LexFortran.cxx 8 lines changed, 4 insertions(+), 4 deletions(-) =================================================================== @@ -481,11 +481,11 @@ static void FoldFortranDoc(Sci_PositionU startPos, Sci_Position length, int init int levelDeltaNext = 0;

const unsigned int nComL = 3; // defines how many comment lines should be before they are folded - Sci_Position nComColB[nComL]; + Sci_Position nComColB[nComL] = {}; Sci_Position nComColF[nComL] = {}; - Sci_Position nComCur; - bool comLineB[nComL]; - bool comLineF[nComL]; + Sci_Position nComCur = 0; + bool comLineB[nComL] = {}; + bool comLineF[nComL] = {}; bool comLineCur; Sci_Position nLineTotal = styler.GetLine(styler.Length()-1) + 1; if (foldComment) {

Modified: scintilla/lexers/LexMarkdown.cxx 60 lines changed, 30 insertions(+), 30 deletions(-) =================================================================== @@ -145,6 +145,7 @@ static void ColorizeMarkdownDoc(Sci_PositionU startPos, Sci_Position length, int WordList **, Accessor &styler) { Sci_PositionU endPos = startPos + length; int precharCount = 0; + bool isLinkNameDetecting = false; // Don't advance on a new loop iteration and retry at the same position. // Useful in the corner case of having to start at the beginning file position // in the default state. @@ -337,45 +338,44 @@ static void ColorizeMarkdownDoc(Sci_PositionU startPos, Sci_Position length, int ++precharCount; }

+ // Any link + if (sc.state == SCE_MARKDOWN_LINK) { + if (sc.Match("](") && sc.GetRelative(-1) != '\') { + sc.Forward(2); + isLinkNameDetecting = true; + } + else if (sc.Match("]:") && sc.GetRelative(-1) != '\') { + sc.Forward(2); + sc.SetState(SCE_MARKDOWN_DEFAULT); + } + else if (!isLinkNameDetecting && sc.ch == ']' && sc.GetRelative(-1) != '\') { + sc.Forward(); + sc.SetState(SCE_MARKDOWN_DEFAULT); + } + else if (isLinkNameDetecting && sc.ch == ')' && sc.GetRelative(-1) != '\') { + sc.Forward(); + sc.SetState(SCE_MARKDOWN_DEFAULT); + isLinkNameDetecting = false; + } + } + // New state anywhere in doc if (sc.state == SCE_MARKDOWN_DEFAULT) { if (sc.atLineStart && sc.ch == '#') { sc.SetState(SCE_MARKDOWN_LINE_BEGIN); freezeCursor = true; } // Links and Images - if (sc.Match("![") || sc.ch == '[') { - Sci_Position i = 0, j = 0, k = 0; - Sci_Position len = endPos - sc.currentPos; - while (i < len && (sc.GetRelative(++i) != ']' || sc.GetRelative(i - 1) == '\')) - ; - if (sc.GetRelative(i) == ']') { - j = i; - if (sc.GetRelative(++i) == '(') { - while (i < len && (sc.GetRelative(++i) != ')' || sc.GetRelative(i - 1) == '\')) - ; - if (sc.GetRelative(i) == ')') - k = i; - } - else if (sc.GetRelative(i) == '[' || sc.GetRelative(++i) == '[') { - while (i < len && (sc.GetRelative(++i) != ']' || sc.GetRelative(i - 1) == '\')) - ; - if (sc.GetRelative(i) == ']') - k = i; - } - } - // At least a link text - if (j) { - sc.SetState(SCE_MARKDOWN_LINK); - sc.Forward(j); - // Also has a URL or reference portion - if (k) - sc.Forward(k - j); - sc.ForwardSetState(SCE_MARKDOWN_DEFAULT); - } + if (sc.Match("![")) { + sc.SetState(SCE_MARKDOWN_LINK); + sc.Forward(2); + } + else if (sc.ch == '[' && sc.GetRelative(-1) != '\') { + sc.SetState(SCE_MARKDOWN_LINK); + sc.Forward(); } // Code - also a special case for alternate inside spacing - if (sc.Match("``") && sc.GetRelative(3) != ' ' && AtTermStart(sc)) { + else if (sc.Match("``") && sc.GetRelative(3) != ' ' && AtTermStart(sc)) { sc.SetState(SCE_MARKDOWN_CODE2); sc.Forward(); }

Modified: scintilla/lexers/LexVerilog.cxx 6 lines changed, 5 insertions(+), 1 deletions(-) =================================================================== @@ -57,7 +57,11 @@ class LinePPState { return level >= 0 && level < 32; } int maskLevel() const { - return 1 << level; + if (level >= 0) { + return 1 << level; + } else { + return 1; + } } public: LinePPState() : state(0), ifTaken(0), level(-1) {

Modified: scintilla/lexlib/CharacterCategory.cxx 240 lines changed, 178 insertions(+), 62 deletions(-) =================================================================== @@ -19,7 +19,7 @@ namespace {

const int catRanges[] = { //++Autogenerated -- start of section automatically generated -// Created with Python 3.6.1, Unicode 9.0.0 +// Created with Python 3.7.0, Unicode 11.0.0 25, 1046, 1073, @@ -651,9 +651,7 @@ const int catRanges[] = { 43773, 43811, 43857, -44061, -44065, -45341, +44033, 45361, 45388, 45437, @@ -672,7 +670,7 @@ const int catRanges[] = { 47389, 47620, 48509, -48644, +48612, 48753, 48829, 49178, @@ -730,6 +728,8 @@ const int catRanges[] = { 65265, 65347, 65405, +65445, +65491, 65540, 66245, 66371, @@ -746,11 +746,13 @@ const int catRanges[] = { 68509, 68561, 68605, +68612, +68989, 70660, 71357, 71364, 71645, -72325, +72293, 72794, 72805, 73830, @@ -812,7 +814,10 @@ const int catRanges[] = { 81546, 81749, 81779, -81821, +81796, +81841, +81861, +81917, 81957, 82022, 82077, @@ -849,7 +854,8 @@ const int catRanges[] = { 85509, 85572, 85669, -85725, +85713, +85757, 86053, 86118, 86173, @@ -886,7 +892,8 @@ const int catRanges[] = { 89651, 89693, 89892, -89949, +89925, +90141, 90149, 90182, 90269, @@ -969,7 +976,7 @@ const int catRanges[] = { 98173, 98309, 98342, -98461, +98437, 98468, 98749, 98756, @@ -1000,7 +1007,7 @@ const int catRanges[] = { 102404, 102437, 102470, -102557, +102545, 102564, 102845, 102852, @@ -1034,15 +1041,15 @@ const int catRanges[] = { 106013, 106020, 106109, -106533, +106501, 106566, 106653, 106660, 106941, 106948, 107069, 107076, -108413, +108389, 108452, 108486, 108581, @@ -1229,10 +1236,11 @@ const int catRanges[] = { 137501, 137632, 137693, -137732, +137729, 139121, 139139, -139172, +139169, +139268, 149821, 149828, 149981, @@ -1337,7 +1345,7 @@ const int catRanges[] = { 197636, 198755, 198788, -200477, +200509, 200708, 200869, 200932, @@ -1463,6 +1471,9 @@ const int catRanges[] = { 233425, 233473, 233789, +233984, +235389, +235424, 235537, 235805, 236037, @@ -1476,7 +1487,7 @@ const int catRanges[] = { 237126, 237189, 237220, -237309, +237286, 237317, 237405, 237569, @@ -1486,7 +1497,7 @@ const int catRanges[] = { 241441, 242531, 243717, -245469, +245597, 245605, 245760, 245793, @@ -1849,7 +1860,7 @@ const int catRanges[] = { 266755, 267197, 267283, -268285, +268317, 268805, 269223, 269349, @@ -1940,8 +1951,6 @@ const int catRanges[] = { 292501, 293778, 293973, -294909, -294933, 296189, 296981, 297341, @@ -2030,13 +2039,9 @@ const int catRanges[] = { 356053, 357085, 357141, -358237, -358325, 358717, 358741, -359005, -359829, -359965, +360445, 360448, 361981, 361985, @@ -2239,7 +2244,7 @@ const int catRanges[] = { 378929, 378957, 378993, -379069, +379389, 380949, 381789, 381813, @@ -2302,7 +2307,7 @@ const int catRanges[] = { 401380, 401437, 401572, -402909, +402973, 402980, 406013, 406037, @@ -2331,7 +2336,7 @@ const int catRanges[] = { 636637, 636949, 638980, -1309405, +1310237, 1310724, 1311395, 1311428, @@ -2563,12 +2568,14 @@ const int catRanges[] = { 1373440, 1373473, 1373504, -1373693, +1373665, 1373696, 1373857, 1373888, 1373921, -1373981, +1373952, +1373985, +1374045, 1375972, 1376003, 1376065, @@ -2606,7 +2613,7 @@ const int catRanges[] = { 1384292, 1384337, 1384356, -1384413, +1384421, 1384456, 1384772, 1385669, @@ -2919,7 +2926,7 @@ const int catRanges[] = { 2121732, 2122762, 2122909, -2123268, +2123172, 2123817, 2123844, 2124105, @@ -3010,12 +3017,12 @@ const int catRanges[] = { 2179748, 2179869, 2179876, -2180765, +2180829, 2180869, 2180989, 2181093, 2181130, -2181405, +2181437, 2181649, 2181949, 2182148, @@ -3054,9 +3061,22 @@ const int catRanges[] = { 2201601, 2203261, 2203466, -2203677, +2203652, +2204805, +2204957, +2205192, +2205533, 2214922, 2215933, +2220036, +2220970, +2221284, +2221341, +2221572, +2222277, +2222634, +2222769, +2222941, 2228230, 2228261, 2228294, @@ -3078,6 +3098,8 @@ const int catRanges[] = { 2234298, 2234321, 2234461, +2234810, +2234845, 2234884, 2235709, 2235912, @@ -3090,7 +3112,9 @@ const int catRanges[] = { 2238141, 2238152, 2238481, -2238621, +2238596, +2238630, +2238717, 2238980, 2240101, 2240145, @@ -3104,7 +3128,7 @@ const int catRanges[] = { 2242534, 2242596, 2242737, -2242885, +2242853, 2242993, 2243037, 2243080, @@ -3160,7 +3184,7 @@ const int catRanges[] = { 2254493, 2254500, 2254685, -2254725, +2254693, 2254756, 2254790, 2254853, @@ -3195,7 +3219,8 @@ const int catRanges[] = { 2263921, 2263965, 2263985, -2264029, +2264005, +2264061, 2265092, 2266630, 2266725, @@ -3249,7 +3274,7 @@ const int catRanges[] = { 2283528, 2283869, 2285572, -2286429, +2286461, 2286501, 2286598, 2286661, @@ -3261,13 +3286,44 @@ const int catRanges[] = { 2287505, 2287605, 2287645, +2293764, +2295174, +2295269, +2295558, +2295589, +2295665, +2295709, 2298880, 2299905, 2300936, 2301258, 2301565, 2301924, 2301981, +2310148, +2310181, +2310500, +2311781, +2311974, +2312004, +2312037, +2312177, +2312421, +2312477, +2312708, +2312741, +2312934, +2312997, +2313092, +2314397, +2314436, +2314565, +2314982, +2315013, +2315089, +2315172, +2315217, +2315389, 2316292, 2318141, 2326532, @@ -3297,6 +3353,45 @@ const int catRanges[] = { 2332294, 2332325, 2332413, +2334724, +2334973, +2334980, +2335069, +2335076, +2336293, +2336509, +2336581, +2336637, +2336645, +2336733, +2336741, +2336964, +2336997, +2337053, +2337288, +2337629, +2337796, +2338013, +2338020, +2338109, +2338116, +2339142, +2339325, +2339333, +2339421, +2339430, +2339493, +2339526, +2339557, +2339588, +2339645, +2339848, +2340189, +2350084, +2350693, +2350758, +2350833, +2350909, 2359300, 2388829, 2392073, @@ -3338,6 +3433,11 @@ const int catRanges[] = { 2977565, 2977700, 2978333, +3000320, +3001345, +3002378, +3003121, +3003261, 3006468, 3008701, 3009028, @@ -3347,13 +3447,15 @@ const int catRanges[] = { 3011171, 3011613, 3013635, -3013693, +3013725, 3014660, -3210685, +3210845, 3211268, 3235453, 3538948, -3539037, +3548157, +3550724, +3563421, 3637252, 3640701, 3640836, @@ -3388,10 +3490,12 @@ const int catRanges[] = { 3819589, 3819701, 3819741, +3824650, +3825309, 3825685, 3828477, 3828746, -3829341, +3829565, 3833856, 3834689, 3835520, @@ -3520,6 +3624,12 @@ const int catRanges[] = { 4008797, 4008913, 4008989, +4034090, +4035989, +4036010, +4036115, +4036138, +4036285, 4046852, 4047005, 4047012, @@ -3603,8 +3713,6 @@ const int catRanges[] = { 4071434, 4071869, 4071957, -4072957, -4072981, 4074909, 4075029, 4076989, @@ -3616,18 +3724,20 @@ const int catRanges[] = { 4081981, 4082197, 4082269, +4082709, +4082909, 4087829, 4095860, 4096021, -4119165, +4119229, 4119573, 4119997, 4120085, -4120317, +4120413, 4120597, 4124317, 4124693, -4127421, +4127549, 4128789, 4129181, 4129301, @@ -3638,22 +3748,26 @@ const int catRanges[] = { 4133149, 4133397, 4134365, +4136981, +4137373, 4137493, -4137981, -4138005, -4138269, -4138517, -4138557, -4138613, 4139005, 4139029, -4139421, -4139541, -4140029, -4141077, -4141661, +4140605, +4140661, +4140797, +4140885, +4140925, +4140949, +4142205, +4142613, +4142941, 4143125, -4143165, +4143229, +4143637, +4145181, +4148245, +4148701, 4194308, 5561085, 5562372, @@ -3662,6 +3776,8 @@ const int catRanges[] = { 5702621, 5702660, 5887069, +5887492, +6126653, 6225924, 6243293, 29360186,

Modified: scintilla/lexlib/PropSetSimple.cxx 2 lines changed, 1 insertions(+), 1 deletions(-) =================================================================== @@ -117,7 +117,7 @@ static int ExpandAllInPlace(const PropSetSimple &props, std::string &withVars, i innerVarStart = withVars.find("$(", varStart+2); }

- std::string var(withVars.c_str(), varStart + 2, varEnd - varStart - 2); + std::string var(withVars, varStart + 2, varEnd - varStart - 2); std::string val = props.Get(var.c_str());

if (blankVars.contains(var.c_str())) {

Modified: scintilla/scintilla_changes.patch 5 lines changed, 4 insertions(+), 1 deletions(-) =================================================================== @@ -62,7 +62,7 @@ diff --git scintilla/src/Catalogue.cxx scintilla/src/Catalogue.cxx index ed47aa8..e58f1ab 100644 --- scintilla/src/Catalogue.cxx +++ scintilla/src/Catalogue.cxx -@@ -77,125 +77,50 @@ int Scintilla_LinkLexers() { +@@ -77,128 +77,50 @@ int Scintilla_LinkLexers() {

//++Autogenerated -- run scripts/LexGen.py to regenerate //**(\tLINK_LEXER(*);\n) @@ -137,6 +137,7 @@ index ed47aa8..e58f1ab 100644 - LINK_LEXER(lmModula); - LINK_LEXER(lmMSSQL); - LINK_LEXER(lmMySQL); +- LINK_LEXER(lmNim); - LINK_LEXER(lmNimrod); - LINK_LEXER(lmNncrontab); + // We use Octave instead of Matlab @@ -164,6 +165,7 @@ index ed47aa8..e58f1ab 100644 - LINK_LEXER(lmRegistry); LINK_LEXER(lmRuby); LINK_LEXER(lmRust); +- LINK_LEXER(lmSAS); - LINK_LEXER(lmScriptol); - LINK_LEXER(lmSmalltalk); - LINK_LEXER(lmSML); @@ -172,6 +174,7 @@ index ed47aa8..e58f1ab 100644 - LINK_LEXER(lmSpice); LINK_LEXER(lmSQL); - LINK_LEXER(lmSrec); +- LINK_LEXER(lmStata); - LINK_LEXER(lmSTTXT); - LINK_LEXER(lmTACL); - LINK_LEXER(lmTADS3);

Modified: scintilla/src/CaseConvert.cxx 16 lines changed, 11 insertions(+), 5 deletions(-) =================================================================== @@ -55,6 +55,7 @@ int symmetricCaseConversionRanges[] = { 1218,1217,7,2, 1233,1232,48,2, 1377,1329,38,1, +4304,7312,43,1, 7681,7680,75,2, 7841,7840,48,2, 7936,7944,8,1, @@ -79,6 +80,7 @@ int symmetricCaseConversionRanges[] = { 66776,66736,36,1, 68800,68736,51,1, 71872,71840,32,1, +93792,93760,32,1, 125218,125184,34,1,

//--Autogenerated -- end of section automatically generated @@ -184,6 +186,9 @@ int symmetricCaseConversions[] = { 1016,1015, 1019,1018, 1231,1216, +4349,7357, +4350,7358, +4351,7359, 7545,42877, 7549,11363, 8017,8025, @@ -233,6 +238,7 @@ int symmetricCaseConversions[] = { 42899,42898, 42933,42932, 42935,42934, +42937,42936, 43859,42931,

//--Autogenerated -- end of section automatically generated @@ -589,27 +595,27 @@ class CaseConverter : public ICaseConverter { } virtual ~CaseConverter() = default; bool Initialised() const { - return characters.size() > 0; + return !characters.empty(); } void Add(int character, const char *conversion) { characterToConversion.emplace_back(character, conversion); } const char *Find(int character) { const std::vector<int>::iterator it = std::lower_bound(characters.begin(), characters.end(), character); if (it == characters.end()) - return 0; + return nullptr; else if (*it == character) return conversions[it - characters.begin()].conversion; else - return 0; + return nullptr; } size_t CaseConvertString(char *converted, size_t sizeConverted, const char *mixed, size_t lenMixed) override { size_t lenConverted = 0; size_t mixedPos = 0; unsigned char bytes[UTF8MaxBytes + 1]{}; while (mixedPos < lenMixed) { const unsigned char leadByte = mixed[mixedPos]; - const char *caseConverted = 0; + const char *caseConverted = nullptr; size_t lenMixedChar = 1; if (UTF8IsAscii(leadByte)) { caseConverted = Find(leadByte); @@ -774,7 +780,7 @@ CaseConverter *ConverterForConversion(enum CaseConversion conversion) { case CaseConversionLower: return &caseConvLow; } - return 0; + return nullptr; }

}

Modified: scintilla/src/Catalogue.cxx 4 lines changed, 2 insertions(+), 2 deletions(-) =================================================================== @@ -33,7 +33,7 @@ const LexerModule *Catalogue::Find(int language) { return lm; } } - return 0; + return nullptr; }

const LexerModule *Catalogue::Find(const char *languageName) { @@ -45,7 +45,7 @@ const LexerModule *Catalogue::Find(const char *languageName) { } } } - return 0; + return nullptr; }

void Catalogue::AddLexerModule(LexerModule *plm) {

Modified: scintilla/src/CellBuffer.cxx 349 lines changed, 340 insertions(+), 9 deletions(-) =================================================================== @@ -7,11 +7,13 @@

#include <cstddef> #include <cstdlib> +#include <cassert> #include <cstring> #include <cstdio> #include <cstdarg>

#include <stdexcept> +#include <string> #include <vector> #include <algorithm> #include <memory> @@ -27,30 +29,115 @@

namespace Scintilla {

+struct CountWidths { + // Measures the number of characters in a string divided into those + // from the Base Multilingual Plane and those from other planes. + Sci::Position countBasePlane; + Sci::Position countOtherPlanes; + CountWidths(Sci::Position countBasePlane_=0, Sci::Position countOtherPlanes_=0) noexcept : + countBasePlane(countBasePlane_), + countOtherPlanes(countOtherPlanes_) { + } + CountWidths operator-() const noexcept { + return CountWidths(-countBasePlane , -countOtherPlanes); + } + Sci::Position WidthUTF32() const noexcept { + // All code points take one code unit in UTF-32. + return countBasePlane + countOtherPlanes; + } + Sci::Position WidthUTF16() const noexcept { + // UTF-16 takes 2 code units for other planes + return countBasePlane + 2 * countOtherPlanes; + } + void CountChar(int lenChar) noexcept { + if (lenChar == 4) { + countOtherPlanes++; + } else { + countBasePlane++; + } + } +}; + class ILineVector { public: virtual void Init() = 0; virtual void SetPerLine(PerLine *pl) = 0; virtual void InsertText(Sci::Line line, Sci::Position delta) = 0; virtual void InsertLine(Sci::Line line, Sci::Position position, bool lineStart) = 0; - virtual void SetLineStart(Sci::Line line, Sci::Position position) = 0; + virtual void SetLineStart(Sci::Line line, Sci::Position position) noexcept = 0; virtual void RemoveLine(Sci::Line line) = 0; virtual Sci::Line Lines() const noexcept = 0; virtual Sci::Line LineFromPosition(Sci::Position pos) const noexcept = 0; virtual Sci::Position LineStart(Sci::Line line) const noexcept = 0; + virtual void InsertCharacters(Sci::Line line, CountWidths delta) = 0; + virtual void SetLineCharactersWidth(Sci::Line line, CountWidths width) = 0; + virtual int LineCharacterIndex() const noexcept = 0; + virtual bool AllocateLineCharacterIndex(int lineCharacterIndex, Sci::Line lines) = 0; + virtual bool ReleaseLineCharacterIndex(int lineCharacterIndex) = 0; + virtual Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const noexcept = 0; + virtual Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const noexcept = 0; virtual ~ILineVector() {} };

}

using namespace Scintilla;

+template <typename POS> +class LineStartIndex { +public: + int refCount; + Partitioning<POS> starts; + + LineStartIndex() : refCount(0), starts(4) { + // Minimal initial allocation + } + // Deleted so LineStartIndex objects can not be copied. + LineStartIndex(const LineStartIndex &) = delete; + LineStartIndex(LineStartIndex &&) = delete; + void operator=(const LineStartIndex &) = delete; + void operator=(LineStartIndex &&) = delete; + virtual ~LineStartIndex() { + starts.DeleteAll(); + } + bool Allocate(Sci::Line lines) { + refCount++; + Sci::Position length = starts.PositionFromPartition(starts.Partitions()); + for (Sci::Line line = starts.Partitions(); line < lines; line++) { + // Produce an ascending sequence that will be filled in with correct widths later + length++; + starts.InsertPartition(static_cast<POS>(line), static_cast<POS>(length)); + } + return refCount == 1; + } + bool Release() { + if (refCount == 1) { + starts.DeleteAll(); + } + refCount--; + return refCount == 0; + } + bool Active() const noexcept { + return refCount > 0; + } + Sci::Position LineWidth(Sci::Line line) const noexcept { + return starts.PositionFromPartition(static_cast<POS>(line) + 1) - + starts.PositionFromPartition(static_cast<POS>(line)); + } + void SetLineWidth(Sci::Line line, Sci::Position width) { + const Sci::Position widthCurrent = LineWidth(line); + starts.InsertText(static_cast<POS>(line), static_cast<POS>(width - widthCurrent)); + } +}; + template <typename POS> class LineVector : public ILineVector { Partitioning<POS> starts; PerLine *perLine; + LineStartIndex<POS> startsUTF16; + LineStartIndex<POS> startsUTF32; public: - LineVector() : starts(256), perLine(0) { + LineVector() : starts(256), perLine(nullptr) { Init(); } // Deleted so LineVector objects can not be copied. @@ -65,26 +152,43 @@ class LineVector : public ILineVector { if (perLine) { perLine->Init(); } - } + startsUTF32.starts.DeleteAll(); + startsUTF16.starts.DeleteAll(); + } void SetPerLine(PerLine *pl) override { perLine = pl; } void InsertText(Sci::Line line, Sci::Position delta) override { starts.InsertText(static_cast<POS>(line), static_cast<POS>(delta)); } void InsertLine(Sci::Line line, Sci::Position position, bool lineStart) override { - starts.InsertPartition(static_cast<POS>(line), static_cast<POS>(position)); + const POS lineAsPos = static_cast<POS>(line); + starts.InsertPartition(lineAsPos, static_cast<POS>(position)); + if (startsUTF32.Active()) { + startsUTF32.starts.InsertPartition(lineAsPos, + static_cast<POS>(startsUTF32.starts.PositionFromPartition(lineAsPos - 1) + 1)); + } + if (startsUTF16.Active()) { + startsUTF16.starts.InsertPartition(lineAsPos, + static_cast<POS>(startsUTF16.starts.PositionFromPartition(lineAsPos - 1) + 1)); + } if (perLine) { if ((line > 0) && lineStart) line--; perLine->InsertLine(line); } } - void SetLineStart(Sci::Line line, Sci::Position position) override { + void SetLineStart(Sci::Line line, Sci::Position position) noexcept override { starts.SetPartitionStartPosition(static_cast<POS>(line), static_cast<POS>(position)); } void RemoveLine(Sci::Line line) override { starts.RemovePartition(static_cast<POS>(line)); + if (startsUTF32.Active()) { + startsUTF32.starts.RemovePartition(static_cast<POS>(line)); + } + if (startsUTF16.Active()) { + startsUTF16.starts.RemovePartition(static_cast<POS>(line)); + } if (perLine) { perLine->RemoveLine(line); } @@ -98,6 +202,71 @@ class LineVector : public ILineVector { Sci::Position LineStart(Sci::Line line) const noexcept override { return starts.PositionFromPartition(static_cast<POS>(line)); } + void InsertCharacters(Sci::Line line, CountWidths delta) override { + if (startsUTF32.Active()) { + startsUTF32.starts.InsertText(static_cast<POS>(line), static_cast<POS>(delta.WidthUTF32())); + } + if (startsUTF16.Active()) { + startsUTF16.starts.InsertText(static_cast<POS>(line), static_cast<POS>(delta.WidthUTF16())); + } + } + void SetLineCharactersWidth(Sci::Line line, CountWidths width) override { + if (startsUTF32.Active()) { + assert(startsUTF32.starts.Partitions() == starts.Partitions()); + startsUTF32.SetLineWidth(line, width.WidthUTF32()); + } + if (startsUTF16.Active()) { + assert(startsUTF16.starts.Partitions() == starts.Partitions()); + startsUTF16.SetLineWidth(line, width.WidthUTF16()); + } + } + + int LineCharacterIndex() const noexcept override { + int retVal = 0; + if (startsUTF32.Active()) { + retVal |= SC_LINECHARACTERINDEX_UTF32; + } + if (startsUTF16.Active()) { + retVal |= SC_LINECHARACTERINDEX_UTF16; + } + return retVal; + } + bool AllocateLineCharacterIndex(int lineCharacterIndex, Sci::Line lines) override { + bool changed = false; + if ((lineCharacterIndex & SC_LINECHARACTERINDEX_UTF32) != 0) { + changed = startsUTF32.Allocate(lines) || changed; + assert(startsUTF32.starts.Partitions() == starts.Partitions()); + } + if ((lineCharacterIndex & SC_LINECHARACTERINDEX_UTF16) != 0) { + changed = startsUTF16.Allocate(lines) || changed; + assert(startsUTF16.starts.Partitions() == starts.Partitions()); + } + return changed; + } + bool ReleaseLineCharacterIndex(int lineCharacterIndex) override { + bool changed = false; + if ((lineCharacterIndex & SC_LINECHARACTERINDEX_UTF32) != 0) { + changed = startsUTF32.Release() || changed; + } + if ((lineCharacterIndex & SC_LINECHARACTERINDEX_UTF16) != 0) { + changed = startsUTF16.Release() || changed; + } + return changed; + } + Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const noexcept override { + if (lineCharacterIndex == SC_LINECHARACTERINDEX_UTF32) { + return startsUTF32.starts.PositionFromPartition(static_cast<POS>(line)); + } else { + return startsUTF16.starts.PositionFromPartition(static_cast<POS>(line)); + } + } + Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const noexcept override { + if (lineCharacterIndex == SC_LINECHARACTERINDEX_UTF32) { + return static_castSci::Line(startsUTF32.starts.PartitionFromPosition(static_cast<POS>(pos))); + } else { + return static_castSci::Line(startsUTF16.starts.PartitionFromPosition(static_cast<POS>(pos))); + } + } };

Action::Action() { @@ -363,6 +532,7 @@ void UndoHistory::CompletedRedoStep() { CellBuffer::CellBuffer(bool hasStyles_, bool largeDocument_) : hasStyles(hasStyles_), largeDocument(largeDocument_) { readOnly = false; + utf8Substance = false; utf8LineEnds = 0; collectingUndo = true; if (largeDocument) @@ -479,7 +649,7 @@ bool CellBuffer::SetStyleFor(Sci::Position position, Sci::Position lengthStyle, const char *CellBuffer::DeleteChars(Sci::Position position, Sci::Position deleteLength, bool &startSequence) { // InsertString and DeleteChars are the bottleneck though which all changes occur PLATFORM_ASSERT(deleteLength > 0); - const char *data = 0; + const char *data = nullptr; if (!readOnly) { if (collectingUndo) { // Save into the undo/redo stack, but only the characters - not the formatting @@ -504,10 +674,19 @@ void CellBuffer::Allocate(Sci::Position newSize) { } }

+void CellBuffer::SetUTF8Substance(bool utf8Substance_) { + if (utf8Substance != utf8Substance_) { + utf8Substance = utf8Substance_; + ResetLineEnds(); + } +} + void CellBuffer::SetLineEndTypes(int utf8LineEnds_) { if (utf8LineEnds != utf8LineEnds_) { + const int indexes = plv->LineCharacterIndex(); utf8LineEnds = utf8LineEnds_; ResetLineEnds(); + AllocateLineCharacterIndex(indexes); } }

@@ -534,6 +713,23 @@ void CellBuffer::SetPerLine(PerLine *pl) { plv->SetPerLine(pl); }

+int CellBuffer::LineCharacterIndex() const noexcept { + return plv->LineCharacterIndex(); +} + +void CellBuffer::AllocateLineCharacterIndex(int lineCharacterIndex) { + if (utf8Substance) { + if (plv->AllocateLineCharacterIndex(lineCharacterIndex, Lines())) { + // Changed so recalculate whole file + RecalculateIndexLineStarts(0, Lines() - 1); + } + } +} + +void CellBuffer::ReleaseLineCharacterIndex(int lineCharacterIndex) { + plv->ReleaseLineCharacterIndex(lineCharacterIndex); +} + Sci::Line CellBuffer::Lines() const noexcept { return plv->Lines(); } @@ -551,6 +747,14 @@ Sci::Line CellBuffer::LineFromPosition(Sci::Position pos) const noexcept { return plv->LineFromPosition(pos); }

+Sci::Position CellBuffer::IndexLineStart(Sci::Line line, int lineCharacterIndex) const noexcept { + return plv->IndexLineStart(line, lineCharacterIndex); +} + +Sci::Line CellBuffer::LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const noexcept { + return plv->LineFromPositionIndex(pos, lineCharacterIndex); +} + bool CellBuffer::IsReadOnly() const { return readOnly; } @@ -611,6 +815,37 @@ bool CellBuffer::UTF8LineEndOverlaps(Sci::Position position) const { return UTF8IsSeparator(bytes) || UTF8IsSeparator(bytes+1) || UTF8IsNEL(bytes+1); }

+bool CellBuffer::UTF8IsCharacterBoundary(Sci::Position position) const { + assert(position >= 0 && position <= Length()); + if (position > 0) { + std::string back; + for (int i = 0; i < UTF8MaxBytes; i++) { + const Sci::Position posBack = position - i; + if (posBack < 0) { + return false; + } + back.insert(0, 1, substance.ValueAt(posBack)); + if (!UTF8IsTrailByte(back.front())) { + if (i > 0) { + // Have reached a non-trail + const int cla = UTF8Classify(reinterpret_cast<const unsigned char*>(back.data()), back.size()); + if ((cla & UTF8MaskInvalid) || (cla != i)) { + return false; + } + } + break; + } + } + } + if (position < Length()) { + const unsigned char fore = substance.ValueAt(position); + if (UTF8IsTrailByte(fore)) { + return false; + } + } + return true; +} + void CellBuffer::ResetLineEnds() { // Reinitialize line data -- too much work to preserve plv->Init(); @@ -647,6 +882,42 @@ void CellBuffer::ResetLineEnds() { } }

+namespace { + +CountWidths CountCharacterWidthsUTF8(const char *s, size_t len) noexcept { + CountWidths cw; + size_t remaining = len; + while (remaining > 0) { + const int utf8Status = UTF8Classify(reinterpret_cast<const unsigned char*>(s), len); + const int lenChar = utf8Status & UTF8MaskWidth; + cw.CountChar(lenChar); + s += lenChar; + remaining -= lenChar; + } + return cw; +} + +} + +bool CellBuffer::MaintainingLineCharacterIndex() const noexcept { + return plv->LineCharacterIndex() != SC_LINECHARACTERINDEX_NONE; +} + +void CellBuffer::RecalculateIndexLineStarts(Sci::Line lineFirst, Sci::Line lineLast) { + std::string text; + Sci::Position posLineEnd = LineStart(lineFirst); + for (Sci::Line line = lineFirst; line <= lineLast; line++) { + // Find line start and end, retrieve text of line, count characters and update line width + const Sci::Position posLineStart = posLineEnd; + posLineEnd = LineStart(line+1); + const Sci::Position width = posLineEnd - posLineStart; + text.resize(width); + GetCharRange(const_cast<char *>(text.data()), posLineStart, width); + const CountWidths cw = CountCharacterWidthsUTF8(text.data(), text.size()); + plv->SetLineCharactersWidth(line, cw); + } +} + void CellBuffer::BasicInsertString(Sci::Position position, const char *s, Sci::Position insertLength) { if (insertLength == 0) return; @@ -658,12 +929,27 @@ void CellBuffer::BasicInsertString(Sci::Position position, const char *s, Sci::P breakingUTF8LineEnd = UTF8LineEndOverlaps(position); }

+ const Sci::Line linePosition = plv->LineFromPosition(position); + Sci::Line lineInsert = linePosition + 1; + + // A simple insertion is one that inserts valid text on a single line at a character boundary + bool simpleInsertion = false; + + const bool maintainingIndex = MaintainingLineCharacterIndex(); + + // Check for breaking apart a UTF-8 sequence and inserting invalid UTF-8 + if (utf8Substance && maintainingIndex) { + // Actually, don't need to check that whole insertion is valid just that there + // are no potential fragments at ends. + simpleInsertion = UTF8IsCharacterBoundary(position) && + UTF8IsValid(s, insertLength); + } + substance.InsertFromArray(position, s, 0, insertLength); if (hasStyles) { style.InsertValue(position, insertLength, 0); }

- Sci::Line lineInsert = plv->LineFromPosition(position) + 1; const bool atLineStart = plv->LineStart(lineInsert-1) == position; // Point all the lines after the insertion point further along in the buffer plv->InsertText(lineInsert-1, insertLength); @@ -683,6 +969,7 @@ void CellBuffer::BasicInsertString(Sci::Position position, const char *s, Sci::P if (ch == '\r') { InsertLine(lineInsert, (position + i) + 1, atLineStart); lineInsert++; + simpleInsertion = false; } else if (ch == '\n') { if (chPrev == '\r') { // Patch up what was end of line @@ -691,11 +978,13 @@ void CellBuffer::BasicInsertString(Sci::Position position, const char *s, Sci::P InsertLine(lineInsert, (position + i) + 1, atLineStart); lineInsert++; } + simpleInsertion = false; } else if (utf8LineEnds) { const unsigned char back3[3] = {chBeforePrev, chPrev, ch}; if (UTF8IsSeparator(back3) || UTF8IsNEL(back3+1)) { InsertLine(lineInsert, (position + i) + 1, atLineStart); lineInsert++; + simpleInsertion = false; } } chBeforePrev = chPrev; @@ -706,6 +995,7 @@ void CellBuffer::BasicInsertString(Sci::Position position, const char *s, Sci::P if (ch == '\r') { // End of line already in buffer so drop the newly created one RemoveLine(lineInsert - 1); + simpleInsertion = false; } } else if (utf8LineEnds && !UTF8IsAscii(chAfter)) { // May have end of UTF-8 line end in buffer and start in insertion @@ -715,21 +1005,33 @@ void CellBuffer::BasicInsertString(Sci::Position position, const char *s, Sci::P if (UTF8IsSeparator(back3)) { InsertLine(lineInsert, (position + insertLength + j) + 1, atLineStart); lineInsert++; + simpleInsertion = false; } if ((j == 0) && UTF8IsNEL(back3+1)) { InsertLine(lineInsert, (position + insertLength + j) + 1, atLineStart); lineInsert++; + simpleInsertion = false; } chBeforePrev = chPrev; chPrev = chAt; } } + if (maintainingIndex) { + if (simpleInsertion) { + const CountWidths cw = CountCharacterWidthsUTF8(s, insertLength); + plv->InsertCharacters(linePosition, cw); + } else { + RecalculateIndexLineStarts(linePosition, lineInsert - 1); + } + } }

void CellBuffer::BasicDeleteChars(Sci::Position position, Sci::Position deleteLength) { if (deleteLength == 0) return;

+ Sci::Line lineRecalculateStart = INVALID_POSITION; + if ((position == 0) && (deleteLength == substance.Length())) { // If whole buffer is being deleted, faster to reinitialise lines data // than to delete each line. @@ -738,11 +1040,37 @@ void CellBuffer::BasicDeleteChars(Sci::Position position, Sci::Position deleteLe // Have to fix up line positions before doing deletion as looking at text in buffer // to work out which lines have been removed

- Sci::Line lineRemove = plv->LineFromPosition(position) + 1; + const Sci::Line linePosition = plv->LineFromPosition(position); + Sci::Line lineRemove = linePosition + 1; + plv->InsertText(lineRemove-1, - (deleteLength)); const unsigned char chPrev = substance.ValueAt(position - 1); const unsigned char chBefore = chPrev; unsigned char chNext = substance.ValueAt(position); + + // Check for breaking apart a UTF-8 sequence + // Needs further checks that text is UTF-8 or that some other break apart is occurring + if (utf8Substance && MaintainingLineCharacterIndex()) { + const Sci::Position posEnd = position + deleteLength; + const Sci::Line lineEndRemove = plv->LineFromPosition(posEnd); + const bool simpleDeletion = + (linePosition == lineEndRemove) && + UTF8IsCharacterBoundary(position) && UTF8IsCharacterBoundary(posEnd); + if (simpleDeletion) { + std::string text(deleteLength, '\0'); + GetCharRange(const_cast<char *>(text.data()), position, deleteLength); + if (UTF8IsValid(text.data(), text.size())) { + // Everything is good + const CountWidths cw = CountCharacterWidthsUTF8(text.data(), text.size()); + plv->InsertCharacters(linePosition, -cw); + } else { + lineRecalculateStart = linePosition; + } + } else { + lineRecalculateStart = linePosition; + } + } + bool ignoreNL = false; if (chPrev == '\r' && chNext == '\n') { // Move back one @@ -791,6 +1119,9 @@ void CellBuffer::BasicDeleteChars(Sci::Position position, Sci::Position deleteLe } } substance.DeleteRange(position, deleteLength); + if (lineRecalculateStart >= 0) { + RecalculateIndexLineStarts(lineRecalculateStart, lineRecalculateStart); + } if (hasStyles) { style.DeleteRange(position, deleteLength); } @@ -816,7 +1147,7 @@ void CellBuffer::EndUndoAction() {

void CellBuffer::AddUndoAction(Sci::Position token, bool mayCoalesce) { bool startSequence; - uh.AppendAction(containerAction, token, 0, 0, startSequence, mayCoalesce); + uh.AppendAction(containerAction, token, nullptr, 0, startSequence, mayCoalesce); }

void CellBuffer::DeleteUndoHistory() {

Modified: scintilla/src/CellBuffer.h 10 lines changed, 10 insertions(+), 0 deletions(-) =================================================================== @@ -113,6 +113,7 @@ class CellBuffer { SplitVector<char> substance; SplitVector<char> style; bool readOnly; + bool utf8Substance; int utf8LineEnds;

bool collectingUndo; @@ -121,7 +122,10 @@ class CellBuffer { std::unique_ptr<ILineVector> plv;

bool UTF8LineEndOverlaps(Sci::Position position) const; + bool UTF8IsCharacterBoundary(Sci::Position position) const; void ResetLineEnds(); + void RecalculateIndexLineStarts(Sci::Line lineFirst, Sci::Line lineLast); + bool MaintainingLineCharacterIndex() const noexcept; /// Actions without undo void BasicInsertString(Sci::Position position, const char *s, Sci::Position insertLength); void BasicDeleteChars(Sci::Position position, Sci::Position deleteLength); @@ -148,13 +152,19 @@ class CellBuffer {

Sci::Position Length() const noexcept; void Allocate(Sci::Position newSize); + void SetUTF8Substance(bool utf8Substance_); int GetLineEndTypes() const { return utf8LineEnds; } void SetLineEndTypes(int utf8LineEnds_); bool ContainsLineEnd(const char *s, Sci::Position length) const; void SetPerLine(PerLine *pl); + int LineCharacterIndex() const noexcept; + void AllocateLineCharacterIndex(int lineCharacterIndex); + void ReleaseLineCharacterIndex(int lineCharacterIndex); Sci::Line Lines() const noexcept; Sci::Position LineStart(Sci::Line line) const noexcept; + Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const noexcept; Sci::Line LineFromPosition(Sci::Position pos) const noexcept; + Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const noexcept; void InsertLine(Sci::Line line, Sci::Position position, bool lineStart); void RemoveLine(Sci::Line line); const char *InsertString(Sci::Position position, const char *s, Sci::Position insertLength, bool &startSequence);

Modified: scintilla/src/ContractionState.cxx 2 lines changed, 1 insertions(+), 1 deletions(-) =================================================================== @@ -210,7 +210,7 @@ Sci::Line ContractionState<LINE>::DocFromDisplay(Sci::Line lineDisplay) const {

template <typename LINE> void ContractionState<LINE>::InsertLines(Sci::Line lineDoc, Sci::Line lineCount) { - for (int l = 0; l < lineCount; l++) { + for (Sci::Line l = 0; l < lineCount; l++) { InsertLine(lineDoc + l); } Check();

Modified: scintilla/src/Document.cxx 102 lines changed, 64 insertions(+), 38 deletions(-) =================================================================== @@ -87,8 +87,31 @@ int LexInterface::LineEndTypesSupported() { return 0; }

+ActionDuration::ActionDuration(double duration_, double minDuration_, double maxDuration_) noexcept : + duration(duration_), minDuration(minDuration_), maxDuration(maxDuration_) { +} + +void ActionDuration::AddSample(size_t numberActions, double durationOfActions) noexcept { + // Only adjust for multiple actions to avoid instability + if (numberActions < 8) + return; + + // Alpha value for exponential smoothing. + // Most recent value contributes 25% to smoothed value. + const double alpha = 0.25; + + const double durationOne = durationOfActions / numberActions; + duration = Sci::clamp(alpha * durationOne + (1.0 - alpha) * duration, + minDuration, maxDuration); +} + +double ActionDuration::Duration() const noexcept { + return duration; +} + Document::Document(int options) : - cb((options & SC_DOCUMENTOPTION_STYLES_NONE) == 0, (options & SC_DOCUMENTOPTION_TEXT_LARGE) != 0) { + cb((options & SC_DOCUMENTOPTION_STYLES_NONE) == 0, (options & SC_DOCUMENTOPTION_TEXT_LARGE) != 0), + durationStyleOneLine(0.00001, 0.000001, 0.0001) { refCount = 0; #ifdef _WIN32 eolMode = SC_EOL_CRLF; @@ -109,7 +132,6 @@ Document::Document(int options) : useTabs = true; tabIndents = true; backspaceUnindents = false; - durationStyleOneLine = 0.00001;

matchesValid = false;

@@ -122,6 +144,7 @@ Document::Document(int options) : decorations = DecorationListCreate(IsLarge());

cb.SetPerLine(this); + cb.SetUTF8Substance(SC_CP_UTF8 == dbcsCodePage); }

Document::~Document() { @@ -197,6 +220,8 @@ bool Document::SetDBCSCodePage(int dbcsCodePage_) { dbcsCodePage = dbcsCodePage_; SetCaseFolder(nullptr); cb.SetLineEndTypes(lineEndBitSet & LineEndTypesSupported()); + cb.SetUTF8Substance(SC_CP_UTF8 == dbcsCodePage); + ModifiedAt(0); // Need to restyle whole document return true; } else { return false; @@ -296,11 +321,11 @@ Sci::Line Document::MarkerNext(Sci::Line lineStart, int mask) const { int Document::AddMark(Sci::Line line, int markerNum) { if (line >= 0 && line <= LinesTotal()) { const int prev = Markers()->AddMark(line, markerNum, LinesTotal()); - const DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, 0, line); + const DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, nullptr, line); NotifyModified(mh); return prev; } else { - return 0; + return -1; } }

@@ -313,19 +338,19 @@ void Document::AddMarkSet(Sci::Line line, int valueSet) { if (m & 1) Markers()->AddMark(line, i, LinesTotal()); } - const DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, 0, line); + const DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, nullptr, line); NotifyModified(mh); }

void Document::DeleteMark(Sci::Line line, int markerNum) { Markers()->DeleteMark(line, markerNum, false); - const DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, 0, line); + const DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, nullptr, line); NotifyModified(mh); }

void Document::DeleteMarkFromHandle(int markerHandle) { Markers()->DeleteMarkFromHandle(markerHandle); - DocModification mh(SC_MOD_CHANGEMARKER, 0, 0, 0, 0); + DocModification mh(SC_MOD_CHANGEMARKER); mh.line = -1; NotifyModified(mh); } @@ -337,7 +362,7 @@ void Document::DeleteAllMarks(int markerNum) { someChanges = true; } if (someChanges) { - DocModification mh(SC_MOD_CHANGEMARKER, 0, 0, 0, 0); + DocModification mh(SC_MOD_CHANGEMARKER); mh.line = -1; NotifyModified(mh); } @@ -423,11 +448,19 @@ Sci::Position Document::VCHomePosition(Sci::Position position) const { return startText; }

+Sci::Position Document::IndexLineStart(Sci::Line line, int lineCharacterIndex) const { + return cb.IndexLineStart(line, lineCharacterIndex); +} + +Sci::Line Document::LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const { + return cb.LineFromPositionIndex(pos, lineCharacterIndex); +} + int SCI_METHOD Document::SetLevel(Sci_Position line, int level) { const int prev = Levels()->SetLevel(static_castSci::Line(line), level, LinesTotal()); if (prev != level) { DocModification mh(SC_MOD_CHANGEFOLD | SC_MOD_CHANGEMARKER, - LineStart(line), 0, 0, 0, static_castSci::Line(line)); + LineStart(line), 0, 0, nullptr, static_castSci::Line(line)); mh.foldLevelNow = level; mh.foldLevelPrev = prev; NotifyModified(mh); @@ -897,8 +930,8 @@ Sci::Position Document::GetRelativePositionUTF16(Sci::Position positionStart, Sc int SCI_METHOD Document::GetCharacterAndWidth(Sci_Position position, Sci_Position *pWidth) const { int character; int bytesInCharacter = 1; + const unsigned char leadByte = cb.UCharAt(position); if (dbcsCodePage) { - const unsigned char leadByte = cb.UCharAt(position); if (SC_CP_UTF8 == dbcsCodePage) { if (UTF8IsAscii(leadByte)) { // Single byte character or invalid @@ -926,7 +959,7 @@ int SCI_METHOD Document::GetCharacterAndWidth(Sci_Position position, Sci_Positio } } } else { - character = cb.CharAt(position); + character = leadByte; } if (pWidth) { *pWidth = bytesInCharacter; @@ -2105,7 +2138,19 @@ const char *Document::SubstituteByPosition(const char *text, Sci::Position *leng if (regex) return regex->SubstituteByPosition(this, text, length); else - return 0; + return nullptr; +} + +int Document::LineCharacterIndex() const { + return cb.LineCharacterIndex(); +} + +void Document::AllocateLineCharacterIndex(int lineCharacterIndex) { + return cb.AllocateLineCharacterIndex(lineCharacterIndex); +} + +void Document::ReleaseLineCharacterIndex(int lineCharacterIndex) { + return cb.ReleaseLineCharacterIndex(lineCharacterIndex); }

Sci::Line Document::LinesTotal() const noexcept { @@ -2191,30 +2236,11 @@ void Document::EnsureStyledTo(Sci::Position pos) { }

void Document::StyleToAdjustingLineDuration(Sci::Position pos) { - // Place bounds on the duration used to avoid glitches spiking it - // and so causing slow styling or non-responsive scrolling - const double minDurationOneLine = 0.000001; - const double maxDurationOneLine = 0.0001; - - // Alpha value for exponential smoothing. - // Most recent value contributes 25% to smoothed value. - const double alpha = 0.25; - const Sci::Line lineFirst = SciLineFromPosition(GetEndStyled()); ElapsedPeriod epStyling; EnsureStyledTo(pos); - const double durationStyling = epStyling.Duration(); const Sci::Line lineLast = SciLineFromPosition(GetEndStyled()); - if (lineLast >= lineFirst + 8) { - // Only adjust for styling multiple lines to avoid instability - const double durationOneLine = durationStyling / (lineLast - lineFirst); - durationStyleOneLine = alpha * durationOneLine + (1.0 - alpha) * durationStyleOneLine; - if (durationStyleOneLine < minDurationOneLine) { - durationStyleOneLine = minDurationOneLine; - } else if (durationStyleOneLine > maxDurationOneLine) { - durationStyleOneLine = maxDurationOneLine; - } - } + durationStyleOneLine.AddSample(lineLast - lineFirst, epStyling.Duration()); }

void Document::LexerChanged() { @@ -2235,7 +2261,7 @@ void Document::SetLexInterface(LexInterface *pLexInterface) { int SCI_METHOD Document::SetLineState(Sci_Position line, int state) { const int statePrevious = States()->SetLineState(static_castSci::Line(line), state); if (state != statePrevious) { - const DocModification mh(SC_MOD_CHANGELINESTATE, LineStart(line), 0, 0, 0, + const DocModification mh(SC_MOD_CHANGELINESTATE, LineStart(line), 0, 0, nullptr, static_castSci::Line(line)); NotifyModified(mh); } @@ -2284,7 +2310,7 @@ void Document::MarginSetStyles(Sci::Line line, const unsigned char *styles) { void Document::MarginClearAll() { const Sci::Line maxEditorLine = LinesTotal(); for (Sci::Line l=0; l<maxEditorLine; l++) - MarginSetText(l, 0); + MarginSetText(l, nullptr); // Free remaining data Margins()->ClearAll(); } @@ -2327,7 +2353,7 @@ int Document::AnnotationLines(Sci::Line line) const { void Document::AnnotationClearAll() { const Sci::Line maxEditorLine = LinesTotal(); for (Sci::Line l=0; l<maxEditorLine; l++) - AnnotationSetText(l, 0); + AnnotationSetText(l, nullptr); // Free remaining data Annotations()->ClearAll(); } @@ -2960,7 +2986,6 @@ std::regex_constants::match_flag_type MatchFlags(const Document *doc, Sci::Posit

template<typename Iterator, typename Regex> bool MatchOnLines(const Document *doc, const Regex &regexp, const RESearchRange &resr, RESearch &search) { - bool matched = false; std::match_results<Iterator> match;

// MSVC and libc++ have problems with ^ and $ matching line ends inside a range. @@ -2974,9 +2999,10 @@ bool MatchOnLines(const Document *doc, const Regex &regexp, const RESearchRange Iterator itStart(doc, resr.startPos); Iterator itEnd(doc, resr.endPos); const std::regex_constants::match_flag_type flagsMatch = MatchFlags(doc, resr.startPos, resr.endPos); - matched = std::regex_search(itStart, itEnd, match, regexp, flagsMatch); + const bool matched = std::regex_search(itStart, itEnd, match, regexp, flagsMatch); #else // Line by line. + bool matched = false; for (Sci::Line line = resr.lineRangeStart; line != resr.lineRangeBreak; line += resr.increment) { const Range lineRange = resr.LineRange(line); Iterator itStart(doc, lineRange.start); @@ -3167,7 +3193,7 @@ const char *BuiltinRegex::SubstituteByPosition(Document *doc, const char *text, substituted.clear(); const DocumentIndexer di(doc, doc->Length()); search.GrabMatches(di); - for (int j = 0; j < *length; j++) { + for (Sci::Position j = 0; j < *length; j++) { if (text[j] == '\') { if (text[j + 1] >= '0' && text[j + 1] <= '9') { const unsigned int patNum = text[j + 1] - '0';

Modified: scintilla/src/Document.h 27 lines changed, 26 insertions(+), 1 deletions(-) =================================================================== @@ -189,6 +189,26 @@ struct RegexError : public std::runtime_error { RegexError() : std::runtime_error("regex failure") {} };

+/** + * The ActionDuration class stores the average time taken for some action such as styling or + * wrapping a line. It is used to decide how many repetitions of that action can be performed + * on idle to maximize efficiency without affecting application responsiveness. + * The duration changes if the time for the action changes. For example, if a simple lexer is + * changed to a complex lexer. Changes are damped and clamped to avoid short periods of easy + * or difficult processing moving the value too far leading to inefficiency or poor user + * experience. + */ + +class ActionDuration { + double duration; + const double minDuration; + const double maxDuration; +public: + ActionDuration(double duration_, double minDuration_, double maxDuration_) noexcept; + void AddSample(size_t numberActions, double durationOfActions) noexcept; + double Duration() const noexcept; +}; + /** */ class Document : PerLine, public IDocumentWithLineEnd, public ILoader { @@ -259,7 +279,7 @@ class Document : PerLine, public IDocumentWithLineEnd, public ILoader { bool useTabs; bool tabIndents; bool backspaceUnindents; - double durationStyleOneLine; + ActionDuration durationStyleOneLine;

std::unique_ptr<IDecorationList> decorations;

@@ -389,6 +409,8 @@ class Document : PerLine, public IDocumentWithLineEnd, public ILoader { bool IsLineEndPosition(Sci::Position position) const; bool IsPositionInLineEnd(Sci::Position position) const; Sci::Position VCHomePosition(Sci::Position position) const; + Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const; + Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const;

int SCI_METHOD SetLevel(Sci_Position line, int level) override; int SCI_METHOD GetLevel(Sci_Position line) const override; @@ -414,6 +436,9 @@ class Document : PerLine, public IDocumentWithLineEnd, public ILoader { void SetCaseFolder(CaseFolder *pcf_); Sci::Position FindText(Sci::Position minPos, Sci::Position maxPos, const char *search, int flags, Sci::Position *length); const char *SubstituteByPosition(const char *text, Sci::Position *length); + int LineCharacterIndex() const; + void AllocateLineCharacterIndex(int lineCharacterIndex); + void ReleaseLineCharacterIndex(int lineCharacterIndex); Sci::Line LinesTotal() const noexcept;

void SetDefaultCharClasses(bool includeWordClass);

Modified: scintilla/src/EditModel.cxx 2 lines changed, 1 insertions(+), 1 deletions(-) =================================================================== @@ -73,5 +73,5 @@ EditModel::EditModel() : braces{} {

EditModel::~EditModel() { pdoc->Release(); - pdoc = 0; + pdoc = nullptr; }

Modified: scintilla/src/EditView.cxx 2 lines changed, 1 insertions(+), 1 deletions(-) =================================================================== @@ -911,7 +911,7 @@ void EditView::DrawEOL(Surface *surface, const EditModel &model, const ViewStyle int alpha = SC_ALPHA_NOALPHA; if (!hideSelection) { const Sci::Position posAfterLineEnd = model.pdoc->LineStart(line + 1); - eolInSelection = (lastSubLine == true) ? model.sel.InSelectionForEOL(posAfterLineEnd) : 0; + eolInSelection = lastSubLine ? model.sel.InSelectionForEOL(posAfterLineEnd) : 0; alpha = (eolInSelection == 1) ? vsDraw.selAlpha : vsDraw.selAdditionalAlpha; }

Modified: scintilla/src/Editor.cxx 106 lines changed, 77 insertions(+), 29 deletions(-) =================================================================== @@ -21,6 +21,7 @@ #include <algorithm> #include <iterator> #include <memory> +#include <chrono>

#include "Platform.h"

@@ -54,6 +55,7 @@ #include "MarginView.h" #include "EditView.h" #include "Editor.h" +#include "ElapsedPeriod.h"

using namespace Scintilla;

@@ -89,7 +91,7 @@ static bool IsLastStep(const DocModification &mh) { }

Timer::Timer() : - ticking(false), ticksToWait(0), tickerID(0) {} + ticking(false), ticksToWait(0), tickerID{} {}

Idler::Idler() : state(false), idlerID(0) {} @@ -103,7 +105,7 @@ static inline bool IsAllSpacesOrTabs(const char *s, unsigned int len) { return true; }

-Editor::Editor() { +Editor::Editor() : durationWrapOneLine(0.00001, 0.000001, 0.0001) { ctrlID = 0;

stylesValid = false; @@ -181,6 +183,7 @@ Editor::Editor() { needIdleStyling = false;

modEventMask = SC_MODEVENTMASKALL; + commandEvents = true;

pdoc->AddWatcher(this, 0);

@@ -206,7 +209,7 @@ void Editor::SetRepresentations() { reprs.Clear();

// C0 control set - const char * const reps[] = { + const char *const reps[] = { "NUL", "SOH", "STX", "ETX", "EOT", "ENQ", "ACK", "BEL", "BS", "HT", "LF", "VT", "FF", "CR", "SO", "SI", "DLE", "DC1", "DC2", "DC3", "DC4", "NAK", "SYN", "ETB", @@ -220,7 +223,7 @@ void Editor::SetRepresentations() { // C1 control set // As well as Unicode mode, ISO-8859-1 should use these if (IsUnicodeMode()) { - const char * const repsC1[] = { + const char *const repsC1[] = { "PAD", "HOP", "BPH", "NBH", "IND", "NEL", "SSA", "ESA", "HTS", "HTJ", "VTS", "PLD", "PLU", "RI", "SS2", "SS3", "DCS", "PU1", "PU2", "STS", "CCH", "MW", "SPA", "EPA", @@ -294,7 +297,7 @@ void Editor::RefreshStyleData() { }

Point Editor::GetVisibleOriginInMain() const { - return Point(0,0); + return Point(0, 0); }

PointDocument Editor::DocumentPointFromView(Point ptView) const { @@ -767,6 +770,7 @@ void Editor::MultipleSelectAdd(AddNumber addNumber) { selectedText.c_str(), searchFlags, &lengthFound); if (pos >= 0) { sel.AddSelection(SelectionRange(pos + lengthFound, pos)); + ContainerNeedsUpdate(SC_UPDATE_SELECTION); ScrollRange(sel.RangeMain()); Redraw(); if (addNumber == addOne) @@ -1532,7 +1536,12 @@ bool Editor::WrapLines(WrapScope ws) { return false; } } else if (ws == WrapScope::wsIdle) { - lineToWrapEnd = lineToWrap + LinesOnScreen() + 100; + // Try to keep time taken by wrapping reasonable so interaction remains smooth. + const double secondsAllowed = 0.01; + const Sci::Line linesInAllowedTime = Sci::clamp( + static_castSci::Line(secondsAllowed / durationWrapOneLine.Duration()), + LinesOnScreen() + 50, static_castSci::Line(0x10000)); + lineToWrapEnd = lineToWrap + linesInAllowedTime; } const Sci::Line lineEndNeedWrap = std::min(wrapPending.end, pdoc->LinesTotal()); lineToWrapEnd = std::min(lineToWrapEnd, lineEndNeedWrap); @@ -1551,13 +1560,16 @@ bool Editor::WrapLines(WrapScope ws) { if (surface) { //Platform::DebugPrintf("Wraplines: scope=%0d need=%0d..%0d perform=%0d..%0d\n", ws, wrapPending.start, wrapPending.end, lineToWrap, lineToWrapEnd);

+ const Sci::Line linesBeingWrapped = lineToWrapEnd - lineToWrap; + ElapsedPeriod epWrapping; while (lineToWrap < lineToWrapEnd) { if (WrapOneLine(surface, lineToWrap)) { wrapOccurred = true; } wrapPending.Wrapped(lineToWrap); lineToWrap++; } + durationWrapOneLine.AddSample(linesBeingWrapped, epWrapping.Duration());

goodTopLine = pcs->DisplayFromDoc(lineDocTop) + std::min( subLineTop, static_castSci::Line(pcs->GetHeight(lineDocTop)-1)); @@ -2674,9 +2686,11 @@ void Editor::NotifyModified(Document *, DocModification mh, void *) {

// If client wants to see this modification if (mh.modificationType & modEventMask) { - if ((mh.modificationType & (SC_MOD_CHANGESTYLE | SC_MOD_CHANGEINDICATOR)) == 0) { - // Real modification made to text of document. - NotifyChange(); // Send EN_CHANGE + if (commandEvents) { + if ((mh.modificationType & (SC_MOD_CHANGESTYLE | SC_MOD_CHANGEINDICATOR)) == 0) { + // Real modification made to text of document. + NotifyChange(); // Send EN_CHANGE + } }

SCNotification scn = {}; @@ -4155,9 +4169,7 @@ std::string Editor::RangeText(Sci::Position start, Sci::Position end) const { if (start < end) { const Sci::Position len = end - start; std::string ret(len, '\0'); - for (int i = 0; i < len; i++) { - ret[i] = pdoc->CharAt(start + i); - } + pdoc->GetCharRange(const_cast<char *>(ret.data()), start, len); return ret; } return std::string(); @@ -4455,7 +4467,7 @@ void Editor::DwellEnd(bool mouseMoved) { void Editor::MouseLeave() { SetHotSpotRange(nullptr); if (!HaveMouseCapture()) { - ptMouseLast = Point(-1,-1); + ptMouseLast = Point(-1, -1); DwellEnd(true); } } @@ -4810,7 +4822,7 @@ void Editor::ButtonMoveWithModifiers(Point pt, unsigned int, int modifiers) { if (hotspot.Valid() && !PointIsHotspot(pt)) SetHotSpotRange(nullptr);

- if (hotSpotClickPos != INVALID_POSITION && PositionFromLocation(pt,true,true) != hotSpotClickPos) { + if (hotSpotClickPos != INVALID_POSITION && PositionFromLocation(pt, true, true) != hotSpotClickPos) { if (inDragDrop == ddNone) { DisplayCursor(Window::cursorText); } @@ -5051,7 +5063,7 @@ Sci::Position Editor::PositionAfterMaxStyling(Sci::Position posMax, bool scrolli // When scrolling, allow less time to ensure responsive const double secondsAllowed = scrolling ? 0.005 : 0.02;

- const Sci::Line linesToStyle = Sci::clamp(static_cast<int>(secondsAllowed / pdoc->durationStyleOneLine), + const Sci::Line linesToStyle = Sci::clamp(static_cast<int>(secondsAllowed / pdoc->durationStyleOneLine.Duration()), 10, 0x10000); const Sci::Line stylingMaxLine = std::min( pdoc->SciLineFromPosition(pdoc->GetEndStyled()) + linesToStyle, @@ -5201,7 +5213,7 @@ void Editor::SetDocPointer(Document *document) { //Platform::DebugPrintf("** %x setdoc to %x\n", pdoc, document); pdoc->RemoveWatcher(this, 0); pdoc->Release(); - if (document == NULL) { + if (!document) { pdoc = new Document(SC_DOCUMENTOPTION_DEFAULT); } else { pdoc = document; @@ -5520,7 +5532,7 @@ void Editor::NeedShown(Sci::Position pos, Sci::Position len) { }

Sci::Position Editor::GetTag(char *tagValue, int tagNumber) { - const char *text = 0; + const char *text = nullptr; Sci::Position length = 0; if ((tagNumber >= 1) && (tagNumber <= 9)) { char name[3] = "\?"; @@ -5763,7 +5775,7 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) { if (wParam == 0) return 0; char *ptr = CharPtrFromSPtr(lParam); - unsigned int iChar = 0; + size_t iChar = 0; for (; iChar < wParam - 1; iChar++) ptr[iChar] = pdoc->CharAt(iChar); ptr[iChar] = '\0'; @@ -5897,7 +5909,7 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) { return selectedText.LengthWithTerminator(); } else { char *ptr = CharPtrFromSPtr(lParam); - unsigned int iChar = 0; + size_t iChar = 0; if (selectedText.Length()) { for (; iChar < selectedText.LengthWithTerminator(); iChar++) ptr[iChar] = selectedText.Data()[iChar]; @@ -6017,6 +6029,11 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) { static_castSci::Position(wParam), lParam), static_castSci::Position(0), pdoc->Length());

+ case SCI_POSITIONRELATIVECODEUNITS: + return Sci::clamp(pdoc->GetRelativePositionUTF16( + static_castSci::Position(wParam), lParam), + static_castSci::Position(0), pdoc->Length()); + case SCI_LINESCROLL: ScrollTo(topLine + static_castSci::Line(lParam)); HorizontalScrollTo(xOffset + static_cast<int>(wParam) * static_cast<int>(vs.spaceWidth)); @@ -6491,15 +6508,15 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {

case SCI_CLEARTABSTOPS: if (view.ClearTabstops(static_castSci::Line(wParam))) { - const DocModification mh(SC_MOD_CHANGETABSTOPS, 0, 0, 0, 0, static_castSci::Line(wParam)); - NotifyModified(pdoc, mh, NULL); + const DocModification mh(SC_MOD_CHANGETABSTOPS, 0, 0, 0, nullptr, static_castSci::Line(wParam)); + NotifyModified(pdoc, mh, nullptr); } break;

case SCI_ADDTABSTOP: if (view.AddTabstop(static_castSci::Line(wParam), static_cast<int>(lParam))) { - const DocModification mh(SC_MOD_CHANGETABSTOPS, 0, 0, 0, 0, static_castSci::Line(wParam)); - NotifyModified(pdoc, mh, NULL); + const DocModification mh(SC_MOD_CHANGETABSTOPS, 0, 0, 0, nullptr, static_castSci::Line(wParam)); + NotifyModified(pdoc, mh, nullptr); } break;

@@ -6773,6 +6790,23 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) { case SCI_GETIMEINTERACTION: return imeInteraction;

+ case SCI_GETLINECHARACTERINDEX: + return pdoc->LineCharacterIndex(); + + case SCI_ALLOCATELINECHARACTERINDEX: + pdoc->AllocateLineCharacterIndex(static_cast<int>(wParam)); + break; + + case SCI_RELEASELINECHARACTERINDEX: + pdoc->ReleaseLineCharacterIndex(static_cast<int>(wParam)); + break; + + case SCI_LINEFROMINDEXPOSITION: + return pdoc->LineFromPositionIndex(static_castSci::Position(wParam), static_cast<int>(lParam)); + + case SCI_INDEXPOSITIONFROMLINE: + return pdoc->IndexLineStart(static_castSci::Line(wParam), static_cast<int>(lParam)); + // Marker definition and setting case SCI_MARKERDEFINE: if (wParam <= MARKER_MAX) { @@ -7384,7 +7418,7 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) { return pdoc->decorations->AllOnFor(static_castSci::Position(wParam));

case SCI_INDICATORVALUEAT: - return pdoc->decorations->ValueAt(static_cast<int>(wParam), static_castSci::Position(lParam)); + return pdoc->decorations->ValueAt(static_cast<int>(wParam), lParam);

case SCI_INDICATORSTART: return pdoc->decorations->Start(static_cast<int>(wParam), lParam); @@ -7522,11 +7556,15 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) { InvalidateStyleRedraw(); break;

- case SCI_SETZOOM: - vs.zoomLevel = static_cast<int>(wParam); - InvalidateStyleRedraw(); - NotifyZoom(); - break; + case SCI_SETZOOM: { + const int zoomLevel = static_cast<int>(wParam); + if (zoomLevel != vs.zoomLevel) { + vs.zoomLevel = zoomLevel; + InvalidateStyleRedraw(); + NotifyZoom(); + } + break; + }

case SCI_GETZOOM: return vs.zoomLevel; @@ -7615,6 +7653,13 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) { case SCI_GETMODEVENTMASK: return modEventMask;

+ case SCI_SETCOMMANDEVENTS: + commandEvents = static_cast<bool>(wParam); + return 0; + + case SCI_GETCOMMANDEVENTS: + return commandEvents; + case SCI_CONVERTEOLS: pdoc->ConvertLineEnds(static_cast<int>(wParam)); SetSelection(sel.MainCaret(), sel.MainAnchor()); // Ensure selection inside document @@ -8179,6 +8224,9 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) { case SCI_COUNTCHARACTERS: return pdoc->CountCharacters(static_castSci::Position(wParam), lParam);

+ case SCI_COUNTCODEUNITS: + return pdoc->CountUTF16(static_castSci::Position(wParam), lParam); + default: return DefWndProc(iMessage, wParam, lParam); }

Modified: scintilla/src/Editor.h 2 lines changed, 2 insertions(+), 0 deletions(-) =================================================================== @@ -229,6 +229,7 @@ class Editor : public EditModel, public DocWatcher { bool needIdleStyling;

int modEventMask; + bool commandEvents;

SelectionText drag;

@@ -249,6 +250,7 @@ class Editor : public EditModel, public DocWatcher {

// Wrapping support WrapPending wrapPending; + ActionDuration durationWrapOneLine;

bool convertPastes;

Modified: scintilla/src/ExternalLexer.cxx 4 lines changed, 3 insertions(+), 1 deletions(-) =================================================================== @@ -64,7 +64,9 @@ LexerLibrary::LexerLibrary(const char *moduleName_) { // Assign a buffer for the lexer name. char lexname[100] = ""; GetLexerName(i, lexname, sizeof(lexname)); - ExternalLexerModule *lex = new ExternalLexerModule(SCLEX_AUTOMATIC, NULL, lexname, NULL); + ExternalLexerModule *lex = new ExternalLexerModule(SCLEX_AUTOMATIC, nullptr, lexname, nullptr); + // This is storing a second reference to lex in the Catalogue as well as in modules. + // TODO: Should use std::shared_ptr or similar to ensure allocation safety. Catalogue::AddLexerModule(lex);

// Remember ExternalLexerModule so we don't leak it

Modified: scintilla/src/Indicator.cxx 7 lines changed, 3 insertions(+), 4 deletions(-) =================================================================== @@ -48,8 +48,7 @@ void Indicator::Draw(Surface *surface, const PRectangle &rc, const PRectangle &r surface->MoveTo(x, irc.top + y); while (x < xLast) { if ((x + 2) > xLast) { - if (xLast > x) - y = 1; + y = 1; x = xLast; } else { x += 2; @@ -61,7 +60,7 @@ void Indicator::Draw(Surface *surface, const PRectangle &rc, const PRectangle &r const PRectangle rcSquiggle = PixelGridAlign(rc);

const int width = std::min(4000, static_cast<int>(rcSquiggle.Width())); - RGBAImage image(width, 3, 1.0, 0); + RGBAImage image(width, 3, 1.0, nullptr); enum { alphaFull = 0xff, alphaSide = 0x2f, alphaSide2=0x5f }; for (int x = 0; x < width; x++) { if (x%2) { @@ -166,7 +165,7 @@ void Indicator::Draw(Surface *surface, const PRectangle &rc, const PRectangle &r IntegerRectangle ircBox(rcBox); // Cap width at 4000 to avoid large allocations when mistakes made const int width = std::min(ircBox.Width(), 4000); - RGBAImage image(width, ircBox.Height(), 1.0, 0); + RGBAImage image(width, ircBox.Height(), 1.0, nullptr); // Draw horizontal lines top and bottom for (int x=0; x<width; x++) { for (int y = 0; y<ircBox.Height(); y += ircBox.Height() - 1) {

Modified: scintilla/src/KeyMap.cxx 4 lines changed, 4 insertions(+), 0 deletions(-) =================================================================== @@ -45,6 +45,10 @@ unsigned int KeyMap::Find(int key, int modifiers) const { return (it == kmap.end()) ? 0 : it->second; }

+const std::map<KeyModifiers, unsigned int> &KeyMap::GetKeyMap() const { + return kmap; +} + #if PLAT_GTK_MACOSX #define OS_X_KEYS 1 #else

Modified: scintilla/src/KeyMap.h 1 lines changed, 1 insertions(+), 0 deletions(-) =================================================================== @@ -56,6 +56,7 @@ class KeyMap { void Clear(); void AssignCmdKey(int key, int modifiers, unsigned int msg); unsigned int Find(int key, int modifiers) const; // 0 returned on failure + const std::map<KeyModifiers, unsigned int> &GetKeyMap() const; };

}

Modified: scintilla/src/PerLine.cxx 4 lines changed, 2 insertions(+), 2 deletions(-) =================================================================== @@ -349,14 +349,14 @@ const char *LineAnnotation::Text(Sci::Line line) const { if (annotations.Length() && (line >= 0) && (line < annotations.Length()) && annotations[line]) return annotations[line].get()+sizeof(AnnotationHeader); else - return 0; + return nullptr; }

const unsigned char *LineAnnotation::Styles(Sci::Line line) const { if (annotations.Length() && (line >= 0) && (line < annotations.Length()) && annotations[line] && MultipleStyles(line)) return reinterpret_cast<unsigned char *>(annotations[line].get() + sizeof(AnnotationHeader) + Length(line)); else - return 0; + return nullptr; }

static char *AllocateAnnotation(int length, int style) {

Modified: scintilla/src/PositionCache.cxx 7 lines changed, 4 insertions(+), 3 deletions(-) =================================================================== @@ -386,13 +386,14 @@ SpecialRepresentations::SpecialRepresentations() { }

void SpecialRepresentations::SetRepresentation(const char *charBytes, const char *value) { - MapRepresentation::iterator it = mapReprs.find(KeyFromString(charBytes, UTF8MaxBytes)); + const unsigned int key = KeyFromString(charBytes, UTF8MaxBytes); + MapRepresentation::iterator it = mapReprs.find(key); if (it == mapReprs.end()) { // New entry so increment for first byte const unsigned char ucStart = charBytes[0]; startByteHasReprs[ucStart]++; } - mapReprs[KeyFromString(charBytes, UTF8MaxBytes)] = Representation(value); + mapReprs[key] = Representation(value); }

void SpecialRepresentations::ClearRepresentation(const char *charBytes) { @@ -574,7 +575,7 @@ PositionCacheEntry::PositionCacheEntry(const PositionCacheEntry &other) : }

void PositionCacheEntry::Set(unsigned int styleNumber_, const char *s_, - unsigned int len_, XYPOSITION *positions_, unsigned int clock_) { + unsigned int len_, const XYPOSITION *positions_, unsigned int clock_) { Clear(); styleNumber = styleNumber_; len = len_;

Modified: scintilla/src/PositionCache.h 2 lines changed, 1 insertions(+), 1 deletions(-) =================================================================== @@ -150,7 +150,7 @@ class PositionCacheEntry { void operator=(const PositionCacheEntry &) = delete; void operator=(PositionCacheEntry &&) = delete; ~PositionCacheEntry(); - void Set(unsigned int styleNumber_, const char *s_, unsigned int len_, XYPOSITION *positions_, unsigned int clock_); + void Set(unsigned int styleNumber_, const char *s_, unsigned int len_, const XYPOSITION *positions_, unsigned int clock_); voi@@ Diff output truncated at 100000 characters. @@

-------------- This E-Mail was brought to you by github_commit_mail.py (Source: https://github.com/geany/infrastructure).