[master] Fix string parsing problems (#202).

tomhrr · Oct 11, 2018 · 5763974 · 5763974
1 parent e813064
commit 5763974
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 20 deletions.
diff --git a/src/dale/Form/Literal/Literal.cpp b/src/dale/Form/Literal/Literal.cpp
@@ -12,6 +12,21 @@
 using namespace dale::ErrorInst;
 
 namespace dale {
+void substituteChars(std::string *str) {
+    size_t pos = 0;
+    while ((pos = str->find("\\n", pos)) != std::string::npos) {
+        str->replace(pos, 2, "\n");
+    }
+    pos = 0;
+    while ((pos = str->find("\\t", pos)) != std::string::npos) {
+        str->replace(pos, 2, "\t");
+    }
+    pos = 0;
+    while ((pos = str->find("\\\\", pos)) != std::string::npos) {
+        str->replace(pos, 2, "\\");
+    }
+}
+
 llvm::Constant *stringLiteralToConstant(Units *units, Type *type,
                                         Node *node, int *size) {
     Context *ctx = units->top()->ctx;
@@ -49,11 +64,8 @@ llvm::Constant *stringLiteralToConstant(Units *units, Type *type,
             return NULL;
         }
 
-        size_t pos = 0;
-        std::string value = t->str_value;
-        while ((pos = value.find("\\n", pos)) != std::string::npos) {
-            value.replace(pos, 2, "\n");
-        }
+        substituteChars(&(t->str_value));
+        std::string value = t->str_value.c_str();
 
         *size = value.size() + 1;
 
@@ -78,11 +90,8 @@ void FormStringLiteralParse(Units *units, Context *ctx,
     std::string var_name;
     units->top()->getUnusedVarName(&var_name);
 
-    size_t pos = 0;
-    std::string value = node->token->str_value;
-    while ((pos = value.find("\\n", pos)) != std::string::npos) {
-        value.replace(pos, 2, "\n");
-    }
+    substituteChars(&(node->token->str_value));
+    std::string value = node->token->str_value.c_str();
 
     int size = value.size() + 1;
     Type *char_array_type = tr->getArrayType(tr->type_char, size);

diff --git a/src/dale/Lexer/Lexer.cpp b/src/dale/Lexer/Lexer.cpp
@@ -180,22 +180,28 @@ bool Lexer::getNextToken(Token *token, Error *error) {
             begin_col_count = end_col_count;
 
             /* Read characters until you hit a double-quote. */
-            while ((c = getchar_()) && c != EOF &&
-                   ((c != '"') ||
-                    (token->str_value.length() &&
-                     (token->str_value[token->str_value.length() - 1] ==
-                      '\\')))) {
-                if (c == '"') {
-                    token->str_value[token->str_value.length() - 1] = c;
-                } else {
-                    token->str_value.push_back(c);
+            while ((c = getchar_()) && c != EOF) {
+                int slash = 0;
+                if (c == '\\') {
+                    c = getchar_();
+                    if (c == EOF) {
+                        goto end;
+                    }
+                    if (c != '"') {
+                        token->str_value.push_back('\\');
+                        end_col_count++;
+                    }
+                } else if (c == '"') {
+                    goto end;
                 }
-                if (c == '\n') {
+                token->str_value.push_back(c);
+                if (!slash && (c == '\n')) {
                     end_line_count++;
                 } else {
                     end_col_count++;
                 }
             }
+            end:
             if (c == EOF) {
                 error->instance = ErrorInst::UnterminatedStringLiteral;
             } else {

diff --git a/t/001basic/136string-chars.t b/t/001basic/136string-chars.t
@@ -0,0 +1,24 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+$ENV{"DALE_TEST_ARGS"} ||= "";
+my $test_dir = $ENV{"DALE_TEST_DIR"} || ".";
+$ENV{PATH} .= ":.";
+
+use Data::Dumper;
+use Test::More tests => 3;
+
+my @res = `dalec $ENV{"DALE_TEST_ARGS"} $test_dir/t/src/string-chars.dt -o string-chars `;
+is_deeply(\@res, [], 'No compilation errors');
+@res = `./string-chars`;
+is($?, 0, 'Program executed successfully');
+
+chomp for @res;
+
+is_deeply(\@res, ["asdf\tasdf\\", "\\"],
+    'Got correct results');
+
+`rm string-chars`;
+
+1;
diff --git a/t/src/string-chars.dt b/t/src/string-chars.dt
@@ -0,0 +1,5 @@
+(import cstdio)
+(def main (fn extern-c int (void)
+  (printf "asdf\tasdf\\\n")
+  (printf "\\")
+  0))