Branch: refs/heads/master
Author: SiegeLord <slabode(a)aim.com>
Committer: SiegeLord <slabode(a)aim.com>
Date: Tue, 29 Jul 2014 15:27:25 UTC
Commit: db977a999d52f34713b11430a3d02fd6386a880a
https://github.com/geany/geany/commit/db977a999d52f34713b11430a3d02fd6386a8…
Log Message:
-----------
Rust: Add keyword
Modified Paths:
--------------
data/filetypes.rust
Modified: data/filetypes.rust
2 lines changed, 1 insertions(+), 1 deletions(-)
===================================================================
@@ -22,7 +22,7 @@ lexerror=error
[keywords]
# all items must be in one line
-primary=alignof as be box break const continue crate do else enum extern false fn for if impl in let loop match mod mut offsetof once priv proc pub pure ref return self sizeof static struct super trait true type typeof unsafe unsized use while yield
+primary=alignof as be box break const continue crate do else enum extern false fn for if impl in let loop match mod mut offsetof once priv proc pub pure ref return self sizeof static struct super trait true type typeof unsafe unsized use virtual while yield
secondary=bool char f32 f64 i16 i32 i64 i8 int str u16 u32 u64 u8 uint
tertiary=Self
--------------
This E-Mail was brought to you by github_commit_mail.py (Source: https://github.com/geany/infrastructure).
Branch: refs/heads/master
Author: SiegeLord <slabode(a)aim.com>
Committer: SiegeLord <slabode(a)aim.com>
Date: Tue, 29 Jul 2014 15:10:14 UTC
Commit: 5e469dc5aeb1982a046703b95893dfd52e5a1174
https://github.com/geany/geany/commit/5e469dc5aeb1982a046703b95893dfd52e5a1…
Log Message:
-----------
Rust: Update comment parsing.
Rust now allows CRLF line endings in source files, which doesn't actually
change any lexing here but the comment was wrong.
Also, the hashbang initial comment has a special case where #![ doesn't count
as a comment (but instead an inner attribute that just happens to be on the
first line).
Modified Paths:
--------------
tagmanager/ctags/rust.c
Modified: tagmanager/ctags/rust.c
19 lines changed, 15 insertions(+), 4 deletions(-)
===================================================================
@@ -188,19 +188,30 @@ static void scanWhitespace (lexerState *lexer)
}
/* Normal line comments start with two /'s and continue until the next \n
- * (NOT any other newline character!). Additionally, a shebang in the beginning
- * of the file also counts as a line comment.
+ * (potentially after a \r). Additionally, a shebang in the beginning of the
+ * file also counts as a line comment as long as it is not this sequence: #![ .
* Block comments start with / followed by a * and end with a * followed by a /.
* Unlike in C/C++ they nest. */
static void scanComments (lexerState *lexer)
{
- /* // or #! */
- if (lexer->next_c == '/' || lexer->next_c == '!')
+ /* // */
+ if (lexer->next_c == '/')
{
advanceNChar(lexer, 2);
while (lexer->cur_c != EOF && lexer->cur_c != '\n')
advanceChar(lexer);
}
+ /* #! */
+ else if (lexer->next_c == '!')
+ {
+ advanceNChar(lexer, 2);
+ /* If it is exactly #![ then it is not a comment, but an attribute */
+ if (lexer->cur_c == '[')
+ return;
+ while (lexer->cur_c != EOF && lexer->cur_c != '\n')
+ advanceChar(lexer);
+ }
+ /* block comment */
else if (lexer->next_c == '*')
{
int level = 1;
--------------
This E-Mail was brought to you by github_commit_mail.py (Source: https://github.com/geany/infrastructure).
Branch: refs/heads/master
Author: SiegeLord <slabode(a)aim.com>
Committer: SiegeLord <slabode(a)aim.com>
Date: Tue, 29 Jul 2014 14:51:36 UTC
Commit: 756344d901afba9e5a51ae099aa7c1fe9ecef019
https://github.com/geany/geany/commit/756344d901afba9e5a51ae099aa7c1fe9ecef…
Log Message:
-----------
Rust: Parse character literals.
These were omitted by mistake. Caused bugs like '"' being interpreted as a
start/end of a string, '}' as the end of a block etc.
Modified Paths:
--------------
tagmanager/ctags/rust.c
Modified: tagmanager/ctags/rust.c
40 lines changed, 40 insertions(+), 0 deletions(-)
===================================================================
@@ -292,6 +292,41 @@ static void scanRawString (lexerState *lexer)
}
}
+/* This deals with character literals: 'n', '\n', '\uFFFF'; and lifetimes:
+ * 'lifetime. We'll use this approximate regexp for the literals:
+ * \' \\ [^']+ \' or \' [^'] \' or \' \\ \' \'. Either way, we'll treat this
+ * token as a string, so it gets preserved as is for function signatures with
+ * lifetimes. */
+static void scanCharacterOrLifetime (lexerState *lexer)
+{
+ vStringClear(lexer->token_str);
+ advanceAndStoreChar(lexer);
+
+ if (lexer->cur_c == '\\')
+ {
+ advanceAndStoreChar(lexer);
+ /* The \' \\ \' \' (literally '\'') case */
+ if (lexer->cur_c == '\'' && lexer->next_c == '\'')
+ {
+ advanceAndStoreChar(lexer);
+ advanceAndStoreChar(lexer);
+ }
+ /* The \' \\ [^']+ \' case */
+ else
+ {
+ while (lexer->cur_c != EOF && lexer->cur_c != '\'')
+ advanceAndStoreChar(lexer);
+ }
+ }
+ /* The \' [^'] \' case */
+ else if (lexer->cur_c != '\'' && lexer->next_c == '\'')
+ {
+ advanceAndStoreChar(lexer);
+ advanceAndStoreChar(lexer);
+ }
+ /* Otherwise it is malformed, or a lifetime */
+}
+
/* Advances the parser one token, optionally skipping whitespace
* (otherwise it is concatenated and returned as a single whitespace token).
* Whitespace is needed to properly render function signatures. Unrecognized
@@ -334,6 +369,11 @@ static int advanceToken (lexerState *lexer, boolean skip_whitspace)
scanRawString(lexer);
return lexer->cur_token = TOKEN_STRING;
}
+ else if (lexer->cur_c == '\'')
+ {
+ scanCharacterOrLifetime(lexer);
+ return lexer->cur_token = TOKEN_STRING;
+ }
else if (isIdentifierStart(lexer->cur_c))
{
scanIdentifier(lexer);
--------------
This E-Mail was brought to you by github_commit_mail.py (Source: https://github.com/geany/infrastructure).
Branch: refs/heads/master
Author: SiegeLord <slabode(a)aim.com>
Committer: SiegeLord <slabode(a)aim.com>
Date: Tue, 29 Jul 2014 13:23:42 UTC
Commit: d96a314a68a6ca779f3e447dd25724df4881a93a
https://github.com/geany/geany/commit/d96a314a68a6ca779f3e447dd25724df4881a…
Log Message:
-----------
Rust: Change/simplify how the string tokens are handled.
Previously, only the string contents were stored in lexerState::token_str (i.e.
not including the delimeters). Now, the delimeters are stored as well, thus
preserving them. This also simplifies the code a bit.
A new function is added to handle the character storage, which is also now
applied for normal identifiers. To that end, the MAX_STRING_LENGTH was boosted
to 256 so that all reasonably sized identifiers may fit.
Modified Paths:
--------------
tagmanager/ctags/rust.c
Modified: tagmanager/ctags/rust.c
49 lines changed, 20 insertions(+), 29 deletions(-)
===================================================================
@@ -24,7 +24,7 @@
/*
* MACROS
*/
-#define MAX_STRING_LENGTH 64
+#define MAX_STRING_LENGTH 256
/*
* DATA DECLARATIONS
@@ -117,9 +117,7 @@ static void writeCurTokenToStr (lexerState *lexer, vString *out_str)
vStringCat(out_str, lexer->token_str);
break;
case TOKEN_STRING:
- vStringPut(out_str, '"');
vStringCat(out_str, lexer->token_str);
- vStringPut(out_str, '"');
break;
case TOKEN_WHITESPACE:
vStringPut(out_str, ' ');
@@ -152,6 +150,14 @@ static void advanceNChar (lexerState *lexer, int n)
advanceChar(lexer);
}
+/* Store the current character in lexerState::token_str if there is space
+ * (set by MAX_STRING_LENGTH), and then read the next character from the file */
+static void advanceAndStoreChar (lexerState *lexer)
+{
+ if (vStringLength(lexer->token_str) < MAX_STRING_LENGTH)
+ vStringPut(lexer->token_str, (char) lexer->cur_c);
+ advanceChar(lexer);
+}
static boolean isWhitespace (int c)
{
@@ -224,8 +230,7 @@ static void scanIdentifier (lexerState *lexer)
vStringClear(lexer->token_str);
do
{
- vStringPut(lexer->token_str, (char) lexer->cur_c);
- advanceChar(lexer);
+ advanceAndStoreChar(lexer);
} while(lexer->cur_c != EOF && isIdentifierContinue(lexer->cur_c));
}
@@ -237,16 +242,14 @@ static void scanIdentifier (lexerState *lexer)
static void scanString (lexerState *lexer)
{
vStringClear(lexer->token_str);
- advanceChar(lexer);
+ advanceAndStoreChar(lexer);
while (lexer->cur_c != EOF && lexer->cur_c != '"')
{
if (lexer->cur_c == '\\' && lexer->next_c == '"')
- advanceChar(lexer);
- if (vStringLength(lexer->token_str) < MAX_STRING_LENGTH)
- vStringPut(lexer->token_str, (char) lexer->cur_c);
- advanceChar(lexer);
+ advanceAndStoreChar(lexer);
+ advanceAndStoreChar(lexer);
}
- advanceChar(lexer);
+ advanceAndStoreChar(lexer);
}
/* Raw strings look like this: r"" or r##""## where the number of
@@ -255,48 +258,36 @@ static void scanRawString (lexerState *lexer)
{
size_t num_initial_hashes = 0;
vStringClear(lexer->token_str);
- advanceChar(lexer);
+ advanceAndStoreChar(lexer);
/* Count how many leading hashes there are */
while (lexer->cur_c == '#')
{
num_initial_hashes++;
- advanceChar(lexer);
+ advanceAndStoreChar(lexer);
}
if (lexer->cur_c != '"')
return;
- advanceChar(lexer);
+ advanceAndStoreChar(lexer);
while (lexer->cur_c != EOF)
{
- if (vStringLength(lexer->token_str) < MAX_STRING_LENGTH)
- vStringPut(lexer->token_str, (char) lexer->cur_c);
/* Count how many trailing hashes there are. If the number is equal or more
* than the number of leading hashes, break. */
if (lexer->cur_c == '"')
{
size_t num_trailing_hashes = 0;
- advanceChar(lexer);
+ advanceAndStoreChar(lexer);
while (lexer->cur_c == '#' && num_trailing_hashes < num_initial_hashes)
{
num_trailing_hashes++;
- if (vStringLength(lexer->token_str) < MAX_STRING_LENGTH)
- vStringPut(lexer->token_str, (char) lexer->cur_c);
- advanceChar(lexer);
+ advanceAndStoreChar(lexer);
}
if (num_trailing_hashes == num_initial_hashes)
- {
- /* Strip the trailing hashes and quotes */
- if (vStringLength(lexer->token_str) < MAX_STRING_LENGTH && vStringLength(lexer->token_str) > num_trailing_hashes + 1)
- {
- lexer->token_str->length = vStringLength(lexer->token_str) - num_trailing_hashes - 1;
- lexer->token_str->buffer[lexer->token_str->length] = '\0';
- }
break;
- }
}
else
{
- advanceChar(lexer);
+ advanceAndStoreChar(lexer);
}
}
}
--------------
This E-Mail was brought to you by github_commit_mail.py (Source: https://github.com/geany/infrastructure).