From a293a41ac77ead050a88d73303fd17f19c75b9ed Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 13 Feb 2012 15:24:05 -0700 Subject: [PATCH] Highlighter stores ScreenLine objects instead of token arrays. --- spec/atom/highlighter-spec.coffee | 64 +++++++++++++++---------------- src/atom/highlighter.coffee | 37 +++++++++--------- src/atom/screen-line.coffee | 6 ++- 3 files changed, 55 insertions(+), 52 deletions(-) diff --git a/spec/atom/highlighter-spec.coffee b/spec/atom/highlighter-spec.coffee index 742835f32..ffe369372 100644 --- a/spec/atom/highlighter-spec.coffee +++ b/spec/atom/highlighter-spec.coffee @@ -11,8 +11,8 @@ describe "Highlighter", -> describe "constructor", -> it "tokenizes all the lines in the buffer", -> - expect(highlighter.tokensForRow(0)[0]).toEqual(type: 'keyword.definition', value: 'var') - expect(highlighter.tokensForRow(11)[1]).toEqual(type: 'keyword', value: 'return') + expect(highlighter.screenLineForRow(0).tokens[0]).toEqual(type: 'keyword.definition', value: 'var') + expect(highlighter.screenLineForRow(11).tokens[1]).toEqual(type: 'keyword', value: 'return') describe "when the buffer changes", -> changeHandler = null @@ -26,11 +26,11 @@ describe "Highlighter", -> range = new Range([0, 0], [2, 0]) buffer.change(range, "foo()\nbar()\n") - expect(highlighter.tokensForRow(0)[0]).toEqual(type: 'identifier', value: 'foo') - expect(highlighter.tokensForRow(1)[0]).toEqual(type: 'identifier', value: 'bar') + expect(highlighter.screenLineForRow(0).tokens[0]).toEqual(type: 'identifier', value: 'foo') + expect(highlighter.screenLineForRow(1).tokens[0]).toEqual(type: 'identifier', value: 'bar') # line 2 is unchanged - expect(highlighter.tokensForRow(2)[1]).toEqual(type: 'keyword', value: 'if') + expect(highlighter.screenLineForRow(2).tokens[1]).toEqual(type: 'keyword', value: 'if') expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] @@ -43,9 +43,9 @@ describe "Highlighter", -> changeHandler.reset() buffer.insert([2, 0], '/*') - expect(highlighter.tokensForRow(3)[0].type).toBe 'comment' - expect(highlighter.tokensForRow(4)[0].type).toBe 'comment' - expect(highlighter.tokensForRow(5)[0].type).toBe 'comment' + expect(highlighter.screenLineForRow(3).tokens[0].type).toBe 'comment' + expect(highlighter.screenLineForRow(4).tokens[0].type).toBe 'comment' + expect(highlighter.screenLineForRow(5).tokens[0].type).toBe 'comment' expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] @@ -57,7 +57,7 @@ describe "Highlighter", -> buffer.insert([5, 0], '*/') buffer.insert([1, 0], 'var ') - expect(highlighter.tokensForRow(1)[0].type).toBe 'comment' + expect(highlighter.screenLineForRow(1).tokens[0].type).toBe 'comment' describe "when lines are both updated and removed", -> it "updates tokens to reflect the removed lines", -> @@ -65,16 +65,16 @@ describe "Highlighter", -> buffer.change(range, "foo()") # previous line 0 remains - expect(highlighter.tokensForRow(0)[0]).toEqual(type: 'keyword.definition', value: 'var') + expect(highlighter.screenLineForRow(0).tokens[0]).toEqual(type: 'keyword.definition', value: 'var') # previous line 3 should be combined with input to form line 1 - expect(highlighter.tokensForRow(1)[0]).toEqual(type: 'identifier', value: 'foo') - expect(highlighter.tokensForRow(1)[6]).toEqual(type: 'identifier', value: 'pivot') + expect(highlighter.screenLineForRow(1).tokens[0]).toEqual(type: 'identifier', value: 'foo') + expect(highlighter.screenLineForRow(1).tokens[6]).toEqual(type: 'identifier', value: 'pivot') # lines below deleted regions should be shifted upward - expect(highlighter.tokensForRow(2)[1]).toEqual(type: 'keyword', value: 'while') - expect(highlighter.tokensForRow(3)[1]).toEqual(type: 'identifier', value: 'current') - expect(highlighter.tokensForRow(4)[3]).toEqual(type: 'keyword.operator', value: '<') + expect(highlighter.screenLineForRow(2).tokens[1]).toEqual(type: 'keyword', value: 'while') + expect(highlighter.screenLineForRow(3).tokens[1]).toEqual(type: 'identifier', value: 'current') + expect(highlighter.screenLineForRow(4).tokens[3]).toEqual(type: 'keyword.operator', value: '<') expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] @@ -86,9 +86,9 @@ describe "Highlighter", -> changeHandler.reset() buffer.change(new Range([2, 0], [3, 0]), '/*') - expect(highlighter.tokensForRow(2)[0].type).toBe 'comment' - expect(highlighter.tokensForRow(3)[0].type).toBe 'comment' - expect(highlighter.tokensForRow(4)[0].type).toBe 'comment' + expect(highlighter.screenLineForRow(2).tokens[0].type).toBe 'comment' + expect(highlighter.screenLineForRow(3).tokens[0].type).toBe 'comment' + expect(highlighter.screenLineForRow(4).tokens[0].type).toBe 'comment' expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] @@ -101,19 +101,19 @@ describe "Highlighter", -> buffer.change(range, "foo()\nbar()\nbaz()\nquux()") # previous line 0 remains - expect(highlighter.tokensForRow(0)[0]).toEqual(type: 'keyword.definition', value: 'var') + expect(highlighter.screenLineForRow(0).tokens[0]).toEqual(type: 'keyword.definition', value: 'var') # 3 new lines inserted - expect(highlighter.tokensForRow(1)[0]).toEqual(type: 'identifier', value: 'foo') - expect(highlighter.tokensForRow(2)[0]).toEqual(type: 'identifier', value: 'bar') - expect(highlighter.tokensForRow(3)[0]).toEqual(type: 'identifier', value: 'baz') + expect(highlighter.screenLineForRow(1).tokens[0]).toEqual(type: 'identifier', value: 'foo') + expect(highlighter.screenLineForRow(2).tokens[0]).toEqual(type: 'identifier', value: 'bar') + expect(highlighter.screenLineForRow(3).tokens[0]).toEqual(type: 'identifier', value: 'baz') # previous line 2 is joined with quux() on line 4 - expect(highlighter.tokensForRow(4)[0]).toEqual(type: 'identifier', value: 'quux') - expect(highlighter.tokensForRow(4)[4]).toEqual(type: 'keyword', value: 'if') + expect(highlighter.screenLineForRow(4).tokens[0]).toEqual(type: 'identifier', value: 'quux') + expect(highlighter.screenLineForRow(4).tokens[4]).toEqual(type: 'keyword', value: 'if') # previous line 3 is pushed down to become line 5 - expect(highlighter.tokensForRow(5)[3]).toEqual(type: 'identifier', value: 'pivot') + expect(highlighter.screenLineForRow(5).tokens[3]).toEqual(type: 'identifier', value: 'pivot') expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] @@ -125,13 +125,13 @@ describe "Highlighter", -> changeHandler.reset() buffer.insert([2, 0], '/*\nabcde\nabcder') - expect(highlighter.tokensForRow(2)[0].type).toBe 'comment' - expect(highlighter.tokensForRow(3)[0].type).toBe 'comment' - expect(highlighter.tokensForRow(4)[0].type).toBe 'comment' - expect(highlighter.tokensForRow(5)[0].type).toBe 'comment' - expect(highlighter.tokensForRow(6)[0].type).toBe 'comment' - expect(highlighter.tokensForRow(7)[0].type).toBe 'comment' - expect(highlighter.tokensForRow(8)[0].type).not.toBe 'comment' + expect(highlighter.screenLineForRow(2).tokens[0].type).toBe 'comment' + expect(highlighter.screenLineForRow(3).tokens[0].type).toBe 'comment' + expect(highlighter.screenLineForRow(4).tokens[0].type).toBe 'comment' + expect(highlighter.screenLineForRow(5).tokens[0].type).toBe 'comment' + expect(highlighter.screenLineForRow(6).tokens[0].type).toBe 'comment' + expect(highlighter.screenLineForRow(7).tokens[0].type).toBe 'comment' + expect(highlighter.screenLineForRow(8).tokens[0].type).not.toBe 'comment' expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] diff --git a/src/atom/highlighter.coffee b/src/atom/highlighter.coffee index 716d7a534..11ff9d67f 100644 --- a/src/atom/highlighter.coffee +++ b/src/atom/highlighter.coffee @@ -6,11 +6,11 @@ module.exports = class Highlighter buffer: null tokenizer: null - lines: [] + screenLines: [] constructor: (@buffer) -> @buildTokenizer() - @lines = @tokenizeRows('start', 0, @buffer.lastRow()) + @screenLines = @buildScreenLinesForRows('start', 0, @buffer.lastRow()) @buffer.on 'change', (e) => @handleBufferChange(e) buildTokenizer: -> @@ -20,11 +20,11 @@ class Highlighter handleBufferChange: (e) -> oldRange = e.oldRange.copy() newRange = e.newRange.copy() - previousState = @lines[oldRange.end.row].state # used in spill detection below + previousState = @screenLines[oldRange.end.row].state # used in spill detection below - startState = @lines[newRange.start.row - 1]?.state or 'start' - @lines[oldRange.start.row..oldRange.end.row] = - @tokenizeRows(startState, newRange.start.row, newRange.end.row) + startState = @screenLines[newRange.start.row - 1]?.state or 'start' + @screenLines[oldRange.start.row..oldRange.end.row] = + @buildScreenLinesForRows(startState, newRange.start.row, newRange.end.row) # spill detection # compare scanner state of last re-highlighted line with its previous state. @@ -32,10 +32,10 @@ class Highlighter # each line until the line's new state matches the previous state. this covers # cases like inserting a /* needing to comment out lines below until we see a */ for row in [newRange.end.row...@buffer.lastRow()] - break if @lines[row].state == previousState + break if @screenLines[row].state == previousState nextRow = row + 1 - previousState = @lines[nextRow].state - @lines[nextRow] = @tokenizeRow(@lines[row].state, nextRow) + previousState = @screenLines[nextRow].state + @screenLines[nextRow] = @buildScreenLineForRow(@screenLines[row].state, nextRow) # if highlighting spilled beyond the bounds of the textual change, update # the pre and post range to reflect area of highlight changes @@ -48,20 +48,19 @@ class Highlighter @trigger("change", {oldRange, newRange}) - tokenizeRows: (startState, startRow, endRow) -> + buildScreenLinesForRows: (startState, startRow, endRow) -> state = startState for row in [startRow..endRow] - line = @tokenizeRow(state, row) - state = line.state - line + screenLine = @buildScreenLineForRow(state, row) + state = screenLine.state + screenLine - tokenizeRow: (state, row) -> - @tokenizer.getLineTokens(@buffer.getLine(row), state) + buildScreenLineForRow: (state, row) -> + line = @buffer.getLine(row) + {tokens, state} = @tokenizer.getLineTokens(line, state) + new ScreenLine(tokens, line, state) screenLineForRow: (row) -> - new ScreenLine(@tokensForRow(row), @buffer.getLine(row)) - - tokensForRow: (row) -> - _.clone(@lines[row].tokens) + @screenLines[row] _.extend(Highlighter.prototype, EventEmitter) diff --git a/src/atom/screen-line.coffee b/src/atom/screen-line.coffee index 9f681b8e3..8e99e59f3 100644 --- a/src/atom/screen-line.coffee +++ b/src/atom/screen-line.coffee @@ -2,7 +2,11 @@ _ = require 'underscore' module.exports = class ScreenLine - constructor: (@tokens, @text) -> + tokens: null + text: null + state: null + + constructor: (@tokens, @text, @state) -> splitAt: (column) -> return [this] if column == 0 or column >= @text.length