Added some CodeDocument and CodeEditorComponent tests and improvements

modules/juce_gui_extra/code_editor/juce_CodeDocument.cpp

@@ -126,21 +126,59 @@ public:
 };
 
 //==============================================================================
-CodeDocument::Iterator::Iterator (const CodeDocument& doc) noexcept : document (&doc) {}
+CodeDocument::Iterator::Iterator (const CodeDocument& doc) noexcept
+    : document (&doc)
+{}
+
+CodeDocument::Iterator::Iterator (CodeDocument::Position p) noexcept
+    : document (p.owner),
+      line (p.getLineNumber()),
+      position (p.getPosition())
+{
+    reinitialiseCharPtr();
+
+    for (int i = 0; i < p.getIndexInLine(); ++i)
+    {
+        charPointer.getAndAdvance();
+
+        if (charPointer.isEmpty())
+        {
+            position -= (p.getIndexInLine() - i);
+            break;
+        }
+    }
+}
+
+CodeDocument::Iterator::Iterator() noexcept
+    : document (nullptr)
+{
+}
 
 CodeDocument::Iterator::~Iterator() noexcept {}
 
+
+bool CodeDocument::Iterator::reinitialiseCharPtr() const
+{
+    /** You're trying to use a default constructed iterator. Bad idea! */
+    jassert (document != nullptr);
+
+    if (charPointer.getAddress() == nullptr)
+    {
+        if (auto* l = document->lines[line])
+            charPointer = l->line.getCharPointer();
+        else
+            return false;
+    }
+
+    return true;
+}
+
 juce_wchar CodeDocument::Iterator::nextChar() noexcept
 {
     for (;;)
     {
-        if (charPointer.getAddress() == nullptr)
+        if (! reinitialiseCharPtr())
-        {
+            return 0;
-            if (auto* l = document->lines[line])
-                charPointer = l->line.getCharPointer();
-            else
-                return 0;
-        }
 
         if (auto result = charPointer.getAndAdvance())
         {
@@ -166,28 +204,31 @@ void CodeDocument::Iterator::skip() noexcept
 
 void CodeDocument::Iterator::skipToEndOfLine() noexcept
 {
-    if (charPointer.getAddress() == nullptr)
+    if (! reinitialiseCharPtr())
-    {
+        return;
-        if (auto* l = document->lines[line])
-            charPointer = l->line.getCharPointer();
-        else
-            return;
-    }
 
     position += (int) charPointer.length();
     ++line;
     charPointer = nullptr;
 }
 
+void CodeDocument::Iterator::skipToStartOfLine() noexcept
+{
+    if (! reinitialiseCharPtr())
+        return;
+
+    if (auto* l = document->lines [line])
+    {
+        auto startPtr = l->line.getCharPointer();
+        position -= (int) startPtr.lengthUpTo (charPointer);
+        charPointer = startPtr;
+    }
+}
+
 juce_wchar CodeDocument::Iterator::peekNextChar() const noexcept
 {
-    if (charPointer.getAddress() == nullptr)
+    if (! reinitialiseCharPtr())
-    {
+        return 0;
-        if (auto* l = document->lines[line])
-            charPointer = l->line.getCharPointer();
-        else
-            return 0;
-    }
 
     if (auto c = *charPointer)
         return c;
@@ -198,6 +239,52 @@ juce_wchar CodeDocument::Iterator::peekNextChar() const noexcept
     return 0;
 }
 
+juce_wchar CodeDocument::Iterator::previousChar() noexcept
+{
+    if (! reinitialiseCharPtr())
+        return 0;
+
+    for (;;)
+    {
+        if (auto* l = document->lines[line])
+        {
+            if (charPointer != l->line.getCharPointer())
+            {
+                --position;
+                --charPointer;
+                break;
+            }
+        }
+
+        if (line == 0)
+            return 0;
+
+        --line;
+
+        if (auto* prev = document->lines[line])
+            charPointer = prev->line.getCharPointer().findTerminatingNull();
+    }
+
+    return *charPointer;
+}
+
+juce_wchar CodeDocument::Iterator::peekPreviousChar() const noexcept
+{
+    if (! reinitialiseCharPtr())
+        return 0;
+
+    if (auto* l = document->lines[line])
+    {
+        if (charPointer != l->line.getCharPointer())
+            return *(charPointer - 1);
+
+        if (auto* prev = document->lines[line - 1])
+            return *(prev->line.getCharPointer().findTerminatingNull() - 1);
+    }
+
+    return 0;
+}
+
 void CodeDocument::Iterator::skipWhitespace() noexcept
 {
     while (CharacterFunctions::isWhitespace (peekNextChar()))
@@ -209,6 +296,40 @@ bool CodeDocument::Iterator::isEOF() const noexcept
     return charPointer.getAddress() == nullptr && line >= document->lines.size();
 }
 
+bool CodeDocument::Iterator::isSOF() const noexcept
+{
+    return position == 0;
+}
+
+CodeDocument::Position CodeDocument::Iterator::toPosition() const
+{
+    if (auto* l = document->lines[line])
+    {
+        reinitialiseCharPtr();
+        int indexInLine = 0;
+        auto linePtr = l->line.getCharPointer();
+
+        while (linePtr != charPointer && ! linePtr.isEmpty())
+        {
+            ++indexInLine;
+            ++linePtr;
+        }
+
+        return CodeDocument::Position (*document, line, indexInLine);
+    }
+
+    if (isEOF())
+    {
+        if (auto* last = document->lines.getLast())
+        {
+            auto lineIndex = document->lines.size() - 1;
+            return CodeDocument::Position (*document, lineIndex, last->lineLength);
+        }
+    }
+
+    return CodeDocument::Position (*document, 0, 0);
+}
+
 //==============================================================================
 CodeDocument::Position::Position() noexcept
 {
@@ -939,4 +1060,239 @@ void CodeDocument::remove (const int startPos, const int endPos, const bool undo
     }
 }
 
+//==============================================================================
+//==============================================================================
+#if JUCE_UNIT_TESTS
+
+struct CodeDocumentTest  : public UnitTest
+{
+    CodeDocumentTest()
+        : UnitTest ("CodeDocument", UnitTestCategories::text)
+    {}
+
+    void runTest() override
+    {
+        const juce::String jabberwocky ("'Twas brillig, and the slithy toves\n"
+                                        "Did gyre and gimble in the wabe;\n"
+                                        "All mimsy were the borogoves,\n"
+                                        "And the mome raths outgrabe.\n\n"
+
+                                        "'Beware the Jabberwock, my son!\n"
+                                        "The jaws that bite, the claws that catch!\n"
+                                        "Beware the Jubjub bird, and shun\n"
+                                        "The frumious Bandersnatch!'");
+
+        {
+            beginTest ("Basic checks");
+            CodeDocument d;
+            d.replaceAllContent (jabberwocky);
+
+            expectEquals (d.getNumLines(), 9);
+            expect (d.getLine (0).startsWith ("'Twas brillig"));
+            expect (d.getLine (2).startsWith ("All mimsy"));
+            expectEquals (d.getLine (4), String ("\n"));
+        }
+
+        {
+            beginTest ("Insert/replace/delete");
+
+            CodeDocument d;
+            d.replaceAllContent (jabberwocky);
+
+            d.insertText (CodeDocument::Position (d, 0, 6), "very ");
+            expect (d.getLine (0).startsWith ("'Twas very brillig"),
+                    "Insert text within a line");
+
+            d.replaceSection (74, 83, "Quite hungry");
+            expectEquals (d.getLine (2), String ("Quite hungry were the borogoves,\n"),
+                          "Replace section at start of line");
+
+            d.replaceSection (11, 18, "cold");
+            expectEquals (d.getLine (0), String ("'Twas very cold, and the slithy toves\n"),
+                          "Replace section within a line");
+
+            d.deleteSection (CodeDocument::Position (d, 2, 0), CodeDocument::Position (d, 2, 6));
+            expectEquals (d.getLine (2), String ("hungry were the borogoves,\n"),
+                          "Delete section within a line");
+
+            d.deleteSection (CodeDocument::Position (d, 2, 6), CodeDocument::Position (d, 5, 11));
+            expectEquals (d.getLine (2), String ("hungry Jabberwock, my son!\n"),
+                          "Delete section across multiple line");
+        }
+
+        {
+            beginTest ("Line splitting and joining");
+
+            CodeDocument d;
+            d.replaceAllContent (jabberwocky);
+            expectEquals (d.getNumLines(), 9);
+
+            const String splitComment ("Adding a newline should split a line into two.");
+            d.insertText (49, "\n");
+
+            expectEquals (d.getNumLines(), 10, splitComment);
+            expectEquals (d.getLine (1), String ("Did gyre and \n"), splitComment);
+            expectEquals (d.getLine (2), String ("gimble in the wabe;\n"), splitComment);
+
+            const String joinComment ("Removing a newline should join two lines.");
+            d.deleteSection (CodeDocument::Position (d, 0, 35),
+                             CodeDocument::Position (d, 1, 0));
+
+            expectEquals (d.getNumLines(), 9, joinComment);
+            expectEquals (d.getLine (0), String ("'Twas brillig, and the slithy tovesDid gyre and \n"), joinComment);
+            expectEquals (d.getLine (1), String ("gimble in the wabe;\n"), joinComment);
+        }
+
+        {
+            beginTest ("Undo/redo");
+
+            CodeDocument d;
+            d.replaceAllContent (jabberwocky);
+            d.newTransaction();
+            d.insertText (30, "INSERT1");
+            d.newTransaction();
+            d.insertText (70, "INSERT2");
+            d.undo();
+
+            expect (d.getAllContent().contains ("INSERT1"), "1st edit should remain.");
+            expect (! d.getAllContent().contains ("INSERT2"), "2nd edit should be undone.");
+
+            d.redo();
+            expect (d.getAllContent().contains ("INSERT2"), "2nd edit should be redone.");
+
+            d.newTransaction();
+            d.deleteSection (25, 90);
+            expect (! d.getAllContent().contains ("INSERT1"), "1st edit should be deleted.");
+            expect (! d.getAllContent().contains ("INSERT2"), "2nd edit should be deleted.");
+            d.undo();
+            expect (d.getAllContent().contains ("INSERT1"), "1st edit should be restored.");
+            expect (d.getAllContent().contains ("INSERT2"), "1st edit should be restored.");
+
+            d.undo();
+            d.undo();
+            expectEquals (d.getAllContent(), jabberwocky, "Original document should be restored.");
+        }
+
+        {
+            beginTest ("Positions");
+
+            CodeDocument d;
+            d.replaceAllContent (jabberwocky);
+
+            {
+                const String comment ("Keeps negative positions inside document.");
+                CodeDocument::Position p1 (d, 0, -3);
+                CodeDocument::Position p2 (d, -8);
+                expectEquals (p1.getLineNumber(), 0, comment);
+                expectEquals (p1.getIndexInLine(), 0, comment);
+                expectEquals (p1.getCharacter(), juce_wchar ('\''), comment);
+                expectEquals (p2.getLineNumber(), 0, comment);
+                expectEquals (p2.getIndexInLine(), 0, comment);
+                expectEquals (p2.getCharacter(), juce_wchar ('\''), comment);
+            }
+
+            {
+                const String comment ("Moving by character handles newlines correctly.");
+                CodeDocument::Position p1 (d, 0, 35);
+                p1.moveBy (1);
+                expectEquals (p1.getLineNumber(), 1, comment);
+                expectEquals (p1.getIndexInLine(), 0, comment);
+                p1.moveBy (75);
+                expectEquals (p1.getLineNumber(), 3, comment);
+            }
+
+            {
+                const String comment1 ("setPositionMaintained tracks position.");
+                const String comment2 ("setPositionMaintained tracks position following undos.");
+
+                CodeDocument::Position p1 (d, 3, 0);
+                p1.setPositionMaintained (true);
+                expectEquals (p1.getCharacter(), juce_wchar ('A'), comment1);
+
+                d.newTransaction();
+                d.insertText (p1, "INSERT1");
+
+                expectEquals (p1.getCharacter(), juce_wchar ('A'), comment1);
+                expectEquals (p1.getLineNumber(), 3, comment1);
+                expectEquals (p1.getIndexInLine(), 7, comment1);
+                d.undo();
+                expectEquals (p1.getIndexInLine(), 0, comment2);
+
+                d.newTransaction();
+                d.insertText (15, "\n");
+
+                expectEquals (p1.getLineNumber(), 4, comment1);
+                d.undo();
+                expectEquals (p1.getLineNumber(), 3, comment2);
+            }
+        }
+
+        {
+            beginTest ("Iterators");
+
+            CodeDocument d;
+            d.replaceAllContent (jabberwocky);
+
+            {
+                const String comment1 ("Basic iteration.");
+                const String comment2 ("Reverse iteration.");
+                const String comment3 ("Reverse iteration stops at doc start.");
+                const String comment4 ("Check iteration across line boundaries.");
+
+                CodeDocument::Iterator it (d);
+                expectEquals (it.peekNextChar(), juce_wchar ('\''), comment1);
+                expectEquals (it.nextChar(), juce_wchar ('\''), comment1);
+                expectEquals (it.nextChar(), juce_wchar ('T'), comment1);
+                expectEquals (it.nextChar(), juce_wchar ('w'), comment1);
+                expectEquals (it.peekNextChar(), juce_wchar ('a'), comment2);
+                expectEquals (it.previousChar(), juce_wchar ('w'), comment2);
+                expectEquals (it.previousChar(), juce_wchar ('T'), comment2);
+                expectEquals (it.previousChar(), juce_wchar ('\''), comment2);
+                expectEquals (it.previousChar(), juce_wchar (0), comment3);
+                expect (it.isSOF(), comment3);
+
+                while (it.peekNextChar() != juce_wchar ('D')) // "Did gyre..."
+                    it.nextChar();
+
+                expectEquals (it.nextChar(), juce_wchar ('D'), comment3);
+                expectEquals (it.peekNextChar(), juce_wchar ('i'), comment3);
+                expectEquals (it.previousChar(), juce_wchar ('D'), comment3);
+                expectEquals (it.previousChar(), juce_wchar ('\n'), comment3);
+                expectEquals (it.previousChar(), juce_wchar ('s'), comment3);
+            }
+
+            {
+                const String comment1 ("Iterator created from CodeDocument::Position objects.");
+                const String comment2 ("CodeDocument::Position created from Iterator objects.");
+                const String comment3 ("CodeDocument::Position created from EOF Iterator objects.");
+
+                CodeDocument::Position p (d, 6, 0); // "The jaws..."
+                CodeDocument::Iterator it (p);
+
+                expectEquals (it.nextChar(), juce_wchar ('T'), comment1);
+                expectEquals (it.nextChar(), juce_wchar ('h'), comment1);
+                expectEquals (it.previousChar(), juce_wchar ('h'), comment1);
+                expectEquals (it.previousChar(), juce_wchar ('T'), comment1);
+                expectEquals (it.previousChar(), juce_wchar ('\n'), comment1);
+                expectEquals (it.previousChar(), juce_wchar ('!'), comment1);
+
+                const auto p2 = it.toPosition();
+                expectEquals (p2.getLineNumber(), 5, comment2);
+                expectEquals (p2.getIndexInLine(), 30, comment2);
+
+                while (! it.isEOF())
+                    it.nextChar();
+
+                const auto p3 = it.toPosition();
+                expectEquals (p3.getLineNumber(), d.getNumLines() - 1, comment3);
+                expectEquals (p3.getIndexInLine(), d.getLine (d.getNumLines() - 1).length(), comment3);
+            }
+        }
+    }
+};
+
+static CodeDocumentTest codeDocumentTests;
+
+#endif
+
 } // namespace juce

modules/juce_gui_extra/code_editor/juce_CodeDocument.h

@@ -184,6 +184,8 @@ public:
         CodeDocument* owner = nullptr;
         int characterPos = 0, line = 0, indexInLine = 0;
         bool positionMaintained = false;
+
+        friend class CodeDocument;
     };
 
     //==============================================================================
@@ -358,19 +360,37 @@ public:
     class JUCE_API  Iterator
     {
     public:
+        /** Creates an uninitialised iterator.
+            Don't attempt to call any methods on this until you've given it an
+            owner document to refer to!
+         */
+        Iterator() noexcept;
+
         Iterator (const CodeDocument& document) noexcept;
-        Iterator (const Iterator&) = default;
+        Iterator (CodeDocument::Position) noexcept;
-        Iterator& operator= (const Iterator&) = default;
         ~Iterator() noexcept;
 
-        /** Reads the next character and returns it.
+        Iterator (const Iterator&) = default;
-            @see peekNextChar
+        Iterator& operator= (const Iterator&) = default;
+
+        /** Reads the next character and returns it. Returns 0 if you try to
+            read past the document's end.
+            @see peekNextChar, previousChar
         */
         juce_wchar nextChar() noexcept;
 
-        /** Reads the next character without advancing the current position. */
+        /** Reads the next character without moving the current position. */
         juce_wchar peekNextChar() const noexcept;
 
+        /** Reads the previous character and returns it. Returns 0 if you try to
+            read past the document's start.
+            @see isSOF, peekPreviousChar, nextChar
+         */
+        juce_wchar previousChar() noexcept;
+
+        /** Reads the next character without moving the current position. */
+        juce_wchar peekPreviousChar() const noexcept;
+
         /** Advances the position by one character. */
         void skip() noexcept;
 
@@ -383,13 +403,24 @@ public:
         /** Skips forward until the next character will be the first character on the next line */
         void skipToEndOfLine() noexcept;
 
+        /** Skips backward until the next character will be the first character on this line */
+        void skipToStartOfLine() noexcept;
+
         /** Returns the line number of the next character. */
         int getLine() const noexcept            { return line; }
 
         /** Returns true if the iterator has reached the end of the document. */
         bool isEOF() const noexcept;
 
+        /** Returns true if the iterator is at the start of the document. */
+        bool isSOF() const noexcept;
+
+        /** Convert this iterator to a CodeDocument::Position. */
+        CodeDocument::Position toPosition() const;
+
     private:
+        bool reinitialiseCharPtr() const;
+
         const CodeDocument* document;
         mutable String::CharPointerType charPointer { nullptr };
         int line = 0, position = 0;

modules/juce_gui_extra/code_editor/juce_CodeEditorComponent.cpp

@@ -550,9 +550,7 @@ void CodeEditorComponent::codeDocumentChanged (const int startIndex, const int e
     const CodeDocument::Position affectedTextStart (document, startIndex);
     const CodeDocument::Position affectedTextEnd (document, endIndex);
 
-    clearCachedIterators (affectedTextStart.getLineNumber());
+    retokenise (startIndex, endIndex);
-
-    rebuildLineTokensAsync();
 
     updateCaretPosition();
     columnToTryToMaintain = -1;
@@ -569,6 +567,16 @@ void CodeEditorComponent::codeDocumentChanged (const int startIndex, const int e
     updateScrollBars();
 }
 
+void CodeEditorComponent::retokenise (int startIndex, int endIndex)
+{
+    const CodeDocument::Position affectedTextStart (document, startIndex);
+    juce::ignoreUnused (endIndex); // Leave room for more efficient impl in future.
+
+    clearCachedIterators (affectedTextStart.getLineNumber());
+
+    rebuildLineTokensAsync();
+}
+
 //==============================================================================
 void CodeEditorComponent::updateCaretPosition()
 {
@@ -629,6 +637,7 @@ void CodeEditorComponent::moveCaretTo (const CodeDocument::Position& newPos, con
     updateCaretPosition();
     scrollToKeepCaretOnScreen();
     updateScrollBars();
+    caretPositionMoved();
 
     if (appCommandManager != nullptr && selectionWasActive != isHighlightActive())
         appCommandManager->commandStatusChanged();
@@ -757,6 +766,7 @@ void CodeEditorComponent::insertText (const String& newText)
             document.insertText (caretPos, newText);
 
         scrollToKeepCaretOnScreen();
+        caretPositionMoved();
     }
 }
 
@@ -1224,6 +1234,10 @@ void CodeEditorComponent::editorViewportPositionChanged()
 {
 }
 
+void CodeEditorComponent::caretPositionMoved()
+{
+}
+
 //==============================================================================
 ApplicationCommandTarget* CodeEditorComponent::getNextCommandTarget()
 {
@@ -1536,7 +1550,7 @@ void CodeEditorComponent::clearCachedIterators (const int firstLineToBeInvalid)
 {
     int i;
     for (i = cachedIterators.size(); --i >= 0;)
-        if (cachedIterators.getUnchecked (i)->getLine() < firstLineToBeInvalid)
+        if (cachedIterators.getUnchecked (i).getLine() < firstLineToBeInvalid)
             break;
 
     cachedIterators.removeRange (jmax (0, i - 1), cachedIterators.size());
@@ -1548,29 +1562,29 @@ void CodeEditorComponent::updateCachedIterators (int maxLineNum)
     const int linesBetweenCachedSources = jmax (10, document.getNumLines() / maxNumCachedPositions);
 
     if (cachedIterators.size() == 0)
-        cachedIterators.add (new CodeDocument::Iterator (document));
+        cachedIterators.add (CodeDocument::Iterator (document));
 
     if (codeTokeniser != nullptr)
     {
         for (;;)
         {
-            auto& last = *cachedIterators.getLast();
+            const auto last = cachedIterators.getLast();
 
             if (last.getLine() >= maxLineNum)
                 break;
 
-            auto* t = new CodeDocument::Iterator (last);
+            cachedIterators.add (CodeDocument::Iterator (last));
-            cachedIterators.add (t);
+            auto& t = cachedIterators.getReference (cachedIterators.size() - 1);
             const int targetLine = jmin (maxLineNum, last.getLine() + linesBetweenCachedSources);
 
             for (;;)
             {
-                codeTokeniser->readNextToken (*t);
+                codeTokeniser->readNextToken (t);
 
-                if (t->getLine() >= targetLine)
+                if (t.getLine() >= targetLine)
                     break;
 
-                if (t->isEOF())
+                if (t.isEOF())
                     return;
             }
         }
@@ -1583,7 +1597,7 @@ void CodeEditorComponent::getIteratorForPosition (int position, CodeDocument::It
     {
         for (int i = cachedIterators.size(); --i >= 0;)
         {
-            auto& t = *cachedIterators.getUnchecked (i);
+            auto& t = cachedIterators.getReference (i);
 
             if (t.getPosition() <= position)
             {

modules/juce_gui_extra/code_editor/juce_CodeEditorComponent.h

@@ -252,6 +252,19 @@ public:
     */
     Colour getColourForTokenType (int tokenType) const;
 
+    /** Rebuilds the syntax highlighting for a section of text.
+
+        This happens automatically any time the CodeDocument is edited, but this
+        method lets you change text colours even when the CodeDocument hasn't changed.
+
+        For example, you could use this to highlight tokens as the cursor moves.
+        To do so you'll need to tell your custom CodeTokeniser where the token you
+        want to highlight is, and make it return a special type of token. Then you
+        should call this method supplying the range of the highlighted text.
+        @see CodeTokeniser
+     */
+    void retokenise (int startIndex, int endIndex);
+
     //==============================================================================
     /** A set of colour IDs to use to change the colour of various aspects of the editor.
 
@@ -287,6 +300,9 @@ public:
     /** Called when the view position is scrolled horizontally or vertically. */
     virtual void editorViewportPositionChanged();
 
+    /** Called when the caret position moves. */
+    virtual void caretPositionMoved();
+
     //==============================================================================
     /** This adds the items to the popup menu.
 
@@ -406,7 +422,7 @@ private:
     void rebuildLineTokensAsync();
     void codeDocumentChanged (int start, int end);
 
-    OwnedArray<CodeDocument::Iterator> cachedIterators;
+    Array<CodeDocument::Iterator> cachedIterators;
     void clearCachedIterators (int firstLineToBeInvalid);
     void updateCachedIterators (int maxLineNum);
     void getIteratorForPosition (int position, CodeDocument::Iterator&);