diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index ecdf2a128..09835a1db 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -23,8 +23,8 @@ describe "TokenizedBuffer", -> describe "tokenization", -> it "tokenizes all the lines in the buffer on construction", -> - expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(type: 'keyword.definition', value: 'var') - expect(tokenizedBuffer.lineForScreenRow(11).tokens[1]).toEqual(type: 'keyword', value: 'return') + expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.type.js']) + expect(tokenizedBuffer.lineForScreenRow(11).tokens[1]).toEqual(value: 'return', scopes: ['source.js', 'keyword.control.js']) describe "when the buffer changes", -> changeHandler = null @@ -34,15 +34,15 @@ describe "TokenizedBuffer", -> tokenizedBuffer.on "change", changeHandler describe "when lines are updated, but none are added or removed", -> - it "updates tokens for each of the changed lines", -> + fit "updates tokens for each of the changed lines", -> range = new Range([0, 0], [2, 0]) - buffer.change(range, "foo()\nbar()\n") + buffer.change(range, "foo()\n7\n") - expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(type: 'identifier', value: 'foo') - expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(type: 'identifier', value: 'bar') + expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js']) + expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js']) # line 2 is unchanged - expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(type: 'keyword', value: 'if') + expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(value: 'if', scopes: ['source.js']) expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] diff --git a/src/app/language-mode.coffee b/src/app/language-mode.coffee index a456c05b6..1022105a1 100644 --- a/src/app/language-mode.coffee +++ b/src/app/language-mode.coffee @@ -1,5 +1,6 @@ AceAdaptor = require 'ace-adaptor' Range = require 'range' +TextMateGrammar = require 'text-mate-grammar' _ = require 'underscore' module.exports = @@ -14,6 +15,7 @@ class LanguageMode constructor: (@editSession) -> @buffer = @editSession.buffer @aceMode = @requireAceMode() + @grammar = TextMateGrammar.grammarForExtension(@editSession.buffer.getExtension()) @aceAdaptor = new AceAdaptor(@editSession) _.adviseBefore @editSession, 'insertText', (text) => @@ -90,6 +92,6 @@ class LanguageMode state = @tokenizedBuffer.stateForRow(bufferRow) @aceMode.autoOutdent(state, @aceAdaptor, bufferRow) - getLineTokens: (line, state) -> - {tokens, state} = @aceMode.getTokenizer().getLineTokens(line, state) + getLineTokens: (line, stack) -> + {tokens, stack} = @grammar.getLineTokens(line, stack) diff --git a/src/app/screen-line.coffee b/src/app/screen-line.coffee index 232390358..3b409f8fb 100644 --- a/src/app/screen-line.coffee +++ b/src/app/screen-line.coffee @@ -3,7 +3,7 @@ Point = require 'point' module.exports = class ScreenLine - state: null + stack: null text: null tokens: null screenDelta: null @@ -16,7 +16,7 @@ class ScreenLine _.extend(this, extraFields) copy: -> - new ScreenLine(@tokens, @text, @screenDelta, @bufferDelta, { @state, @foldable }) + new ScreenLine(@tokens, @text, @screenDelta, @bufferDelta, { @stack, @foldable }) splitAt: (column) -> return [new ScreenLine([], '', [0, 0], [0, 0]), this] if column == 0 @@ -37,8 +37,8 @@ class ScreenLine [leftScreenDelta, rightScreenDelta] = @screenDelta.splitAt(column) [leftBufferDelta, rightBufferDelta] = @bufferDelta.splitAt(column) - leftFragment = new ScreenLine(leftTokens, leftText, leftScreenDelta, leftBufferDelta, {@state, @foldable}) - rightFragment = new ScreenLine(rightTokens, rightText, rightScreenDelta, rightBufferDelta, {@state}) + leftFragment = new ScreenLine(leftTokens, leftText, leftScreenDelta, leftBufferDelta, {@stack, @foldable}) + rightFragment = new ScreenLine(rightTokens, rightText, rightScreenDelta, rightBufferDelta, {@stack}) [leftFragment, rightFragment] concat: (other) -> @@ -46,7 +46,7 @@ class ScreenLine text = @text + other.text screenDelta = @screenDelta.add(other.screenDelta) bufferDelta = @bufferDelta.add(other.bufferDelta) - new ScreenLine(tokens, text, screenDelta, bufferDelta, {state: other.state}) + new ScreenLine(tokens, text, screenDelta, bufferDelta, {stack: other.stack}) translateColumn: (sourceDeltaType, targetDeltaType, sourceColumn, options={}) -> { skipAtomicTokens } = options diff --git a/src/app/token.coffee b/src/app/token.coffee index e581ba773..935d584b3 100644 --- a/src/app/token.coffee +++ b/src/app/token.coffee @@ -1,20 +1,22 @@ +_ = require 'underscore' + module.exports = class Token value: null - type: null + scopes: null isAtomic: null - constructor: ({@value, @type, @isAtomic, @bufferDelta, @fold}) -> + constructor: ({@value, @scopes, @isAtomic, @bufferDelta, @fold}) -> @screenDelta = @value.length @bufferDelta ?= @screenDelta isEqual: (other) -> - @value == other.value and @type == other.type and !!@isAtomic == !!other.isAtomic + @value == other.value and _.isEqual(@scopes, other.scopes) and !!@isAtomic == !!other.isAtomic splitAt: (splitIndex) -> value1 = @value.substring(0, splitIndex) value2 = @value.substring(splitIndex) - [new Token(value: value1, type: @type), new Token(value: value2, type: @type)] + [new Token(value: value1, scopes: @scopes), new Token(value: value2, scopes: @scopes)] breakOutTabCharacters: (tabText) -> return [this] unless /\t/.test(@value) @@ -23,7 +25,7 @@ class Token if substring == '\t' @buildTabToken(tabText) else - new Token(value: substring, type: @type) + new Token(value: substring, scopes: @scopes) buildTabToken: (tabText) -> - new Token(value: tabText, type: @type, bufferDelta: 1, isAtomic: true) + new Token(value: tabText, scopes: @scopes, bufferDelta: 1, isAtomic: true) diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index a028a6707..2e8939f5e 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -17,7 +17,7 @@ class TokenizedBuffer constructor: (@buffer, { @languageMode, @tabText }) -> @languageMode.tokenizedBuffer = this @id = @constructor.idCounter++ - @screenLines = @buildScreenLinesForRows('start', 0, @buffer.getLastRow()) + @screenLines = @buildScreenLinesForRows(0, @buffer.getLastRow()) @buffer.on "change.tokenized-buffer#{@id}", (e) => @handleBufferChange(e) handleBufferChange: (e) -> @@ -25,9 +25,9 @@ class TokenizedBuffer newRange = e.newRange.copy() previousState = @stateForRow(oldRange.end.row) # used in spill detection below - startState = @stateForRow(newRange.start.row - 1) + stack = @stateForRow(newRange.start.row - 1) @screenLines[oldRange.start.row..oldRange.end.row] = - @buildScreenLinesForRows(startState, newRange.start.row, newRange.end.row) + @buildScreenLinesForRows(newRange.start.row, newRange.end.row, stack) # spill detection # compare scanner state of last re-highlighted line with its previous state. @@ -38,7 +38,7 @@ class TokenizedBuffer break if @stateForRow(row) == previousState nextRow = row + 1 previousState = @stateForRow(nextRow) - @screenLines[nextRow] = @buildScreenLineForRow(@stateForRow(row), nextRow) + @screenLines[nextRow] = @buildScreenLineForRow(nextRow, @stateForRow(row)) # if highlighting spilled beyond the bounds of the textual change, update # the pre and post range to reflect area of highlight changes @@ -51,22 +51,22 @@ class TokenizedBuffer @trigger("change", {oldRange, newRange}) - buildScreenLinesForRows: (startState, startRow, endRow) -> - state = startState + buildScreenLinesForRows: (startRow, endRow, startingStack) -> + stack = startingStack for row in [startRow..endRow] - screenLine = @buildScreenLineForRow(state, row) - state = screenLine.state + screenLine = @buildScreenLineForRow(row, stack) + stack = screenLine.stack screenLine - buildScreenLineForRow: (state, row) -> + buildScreenLineForRow: (row, stack) -> line = @buffer.lineForRow(row) - {tokens, state} = @languageMode.getLineTokens(line, state) + {tokens, stack} = @languageMode.getLineTokens(line, stack) tokenObjects = [] for tokenProperties in tokens token = new Token(tokenProperties) tokenObjects.push(token.breakOutTabCharacters(@tabText)...) text = _.pluck(tokenObjects, 'value').join('') - new ScreenLine(tokenObjects, text, [1, 0], [1, 0], { state }) + new ScreenLine(tokenObjects, text, [1, 0], [1, 0], { stack }) lineForScreenRow: (row) -> @screenLines[row] @@ -75,7 +75,7 @@ class TokenizedBuffer @screenLines[startRow..endRow] stateForRow: (row) -> - @screenLines[row]?.state ? 'start' + @screenLines[row]?.stack destroy: -> @buffer.off ".tokenized-buffer#{@id}"