github/atom
1
1
Fork 0
mirror of https://github.com/atom/atom.git synced 2026-01-24 06:18:03 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9162 from atom/as-cjk-soft-wrap

Browse Source 
CJK soft wrap
This commit is contained in:
Nathan Sobo
2015-10-16 10:34:52 -06:00
parent 4df007eb87 cd47c7be02
commit 30f45d3146
13 changed files with 255 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
spec/display-buffer-spec.coffee
View File

@@ -60,11 +60,42 @@ describe "DisplayBuffer", ->
describe "soft wrapping", ->
beforeEach ->
displayBuffer.setSoftWrapped(true)
displayBuffer.setEditorWidthInChars(50)
displayBuffer.setSoftWrapped(true)
displayBuffer.setDefaultCharWidth(1)
changeHandler.reset()
describe "rendering of soft-wrapped lines", ->
describe "when there are double width characters", ->
it "takes them into account when finding the soft wrap column", ->
buffer.setText("私たちのフ是一个地方，数千名学生12345业余爱们的板作为hello world this is a pretty long latin line")
displayBuffer.setDefaultCharWidth(1, 5, 0, 0)
expect(displayBuffer.tokenizedLineForScreenRow(0).text).toBe("私たちのフ是一个地方")
expect(displayBuffer.tokenizedLineForScreenRow(1).text).toBe("，数千名学生12345业余爱")
expect(displayBuffer.tokenizedLineForScreenRow(2).text).toBe("们的板作为hello world this is a ")
expect(displayBuffer.tokenizedLineForScreenRow(3).text).toBe("pretty long latin line")
describe "when there are half width characters", ->
it "takes them into account when finding the soft wrap column", ->
displayBuffer.setDefaultCharWidth(1, 0, 5, 0)
buffer.setText("abcﾪﾫﾬﾈﾇﾈﾉﾊﾋﾌﾋﾌﾇﾴ￮￮￮hello world this is a pretty long line")
expect(displayBuffer.tokenizedLineForScreenRow(0).text).toBe("abcﾪﾫﾬﾈﾇﾈﾉﾊﾋ")
expect(displayBuffer.tokenizedLineForScreenRow(1).text).toBe("ﾌﾋﾌﾇﾴ￮￮￮hello ")
expect(displayBuffer.tokenizedLineForScreenRow(2).text).toBe("world this is a pretty long line")
describe "when there are korean characters", ->
it "takes them into account when finding the soft wrap column", ->
displayBuffer.setDefaultCharWidth(1, 0, 0, 10)
buffer.setText("1234세계를 향한 대화, 유니코 제10회유니코드국제")
expect(displayBuffer.tokenizedLineForScreenRow(0).text).toBe("1234세계를 ")
expect(displayBuffer.tokenizedLineForScreenRow(1).text).toBe("향한 대화, ")
expect(displayBuffer.tokenizedLineForScreenRow(2).text).toBe("유니코 ")
expect(displayBuffer.tokenizedLineForScreenRow(3).text).toBe("제10회유니")
expect(displayBuffer.tokenizedLineForScreenRow(4).text).toBe("코드국제")
describe "when editor.softWrapAtPreferredLineLength is set", ->
it "uses the preferred line length as the soft wrap column when it is less than the configured soft wrap column", ->
atom.config.set('editor.preferredLineLength', 100)
@@ -242,6 +273,7 @@ describe "DisplayBuffer", ->
buffer, tabLength, editorWidthInChars: 30, config: atom.config,
grammarRegistry: atom.grammars, packageManager: atom.packages, assert: ->
})
displayBuffer.setDefaultCharWidth(1)
displayBuffer.setSoftWrapped(true)
buffer.insert([0, 0], "the quick brown fox jumps over the lazy dog.")
@@ -651,6 +683,7 @@ describe "DisplayBuffer", ->
beforeEach ->
tabLength = 4
displayBuffer.setDefaultCharWidth(1)
displayBuffer.setTabLength(tabLength)
displayBuffer.setSoftWrapped(true)
displayBuffer.setEditorWidthInChars(50)
@@ -768,6 +801,7 @@ describe "DisplayBuffer", ->
it "correctly translates positions on soft wrapped lines containing tabs", ->
buffer.setText('\t\taa bb cc dd ee ff gg')
displayBuffer.setSoftWrapped(true)
displayBuffer.setDefaultCharWidth(1)
displayBuffer.setEditorWidthInChars(10)
expect(displayBuffer.screenPositionForBufferPosition([0, 10], wrapAtSoftNewlines: true)).toEqual [1, 4]
expect(displayBuffer.bufferPositionForScreenPosition([1, 0])).toEqual [0, 9]

14
spec/text-editor-component-spec.coffee
View File

@@ -2895,6 +2895,18 @@ describe "TextEditorComponent", ->
expect(editor.consolidateSelections).toHaveBeenCalled()
expect(event.abortKeyBinding).toHaveBeenCalled()
describe "when changing the font", ->
it "measures the default char, the korean char, the double width char and the half width char widths", ->
expect(editor.getDefaultCharWidth()).toBeCloseTo(12, 0)
component.setFontSize(10)
nextAnimationFrame()
expect(editor.getDefaultCharWidth()).toBeCloseTo(6, 0)
expect(editor.getKoreanCharWidth()).toBeCloseTo(9, 0)
expect(editor.getDoubleWidthCharWidth()).toBe(10)
expect(editor.getHalfWidthCharWidth()).toBe(5)
describe "hiding and showing the editor", ->
describe "when the editor is hidden when it is mounted", ->
it "defers measurement and rendering until the editor becomes visible", ->
@@ -3212,7 +3224,9 @@ describe "TextEditorComponent", ->
atom.config.set 'editor.preferredLineLength', 17, scopeSelector: '.source.coffee'
atom.config.set 'editor.softWrapAtPreferredLineLength', true, scopeSelector: '.source.coffee'
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(20)
coffeeEditor.setDefaultCharWidth(1)
coffeeEditor.setEditorWidthInChars(20)
it "wraps lines when editor.softWrap is true for a matching scope", ->

5
spec/text-editor-presenter-spec.coffee
View File

@@ -1259,6 +1259,7 @@ describe "TextEditorPresenter", ->
it "only applies decorations to screen rows that are spanned by their marker when lines are soft-wrapped", ->
editor.setText("a line that wraps, ok")
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(16)
marker = editor.markBufferRange([[0, 0], [0, 2]])
editor.decorateMarker(marker, type: 'line', class: 'a')
@@ -2244,6 +2245,7 @@ describe "TextEditorPresenter", ->
it "contains states for line numbers that are visible on screen", ->
editor.foldBufferRow(4)
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(50)
presenter = buildPresenter(explicitHeight: 25, scrollTop: 30, lineHeight: 10, tileSize: 2)
@@ -2259,6 +2261,7 @@ describe "TextEditorPresenter", ->
it "updates when the editor's content changes", ->
editor.foldBufferRow(4)
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(50)
presenter = buildPresenter(explicitHeight: 35, scrollTop: 30, tileSize: 2)
@@ -2289,6 +2292,7 @@ describe "TextEditorPresenter", ->
it "correctly handles the first screen line being soft-wrapped", ->
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(30)
presenter = buildPresenter(explicitHeight: 25, scrollTop: 50, tileSize: 2)
@@ -2417,6 +2421,7 @@ describe "TextEditorPresenter", ->
it "only applies line-number decorations to screen rows that are spanned by their marker when lines are soft-wrapped", ->
editor.setText("a line that wraps, ok")
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(16)
marker = editor.markBufferRange([[0, 0], [0, 2]])
editor.decorateMarker(marker, type: 'line-number', class: 'a')

13
spec/text-editor-spec.coffee
View File

@@ -272,6 +272,7 @@ describe "TextEditor", ->
describe "when soft-wrap is enabled and code is folded", ->
beforeEach ->
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(50)
editor.createFold(2, 3)
@@ -327,6 +328,7 @@ describe "TextEditor", ->
describe "when the cursor was moved down from the beginning of an indented soft-wrapped line", ->
it "moves to the beginning of the previous line", ->
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(50)
editor.setCursorScreenPosition([3, 0])
@@ -379,6 +381,7 @@ describe "TextEditor", ->
describe "when the cursor is at the beginning of an indented soft-wrapped line", ->
it "moves to the beginning of the line's continuation on the next screen row", ->
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(50)
editor.setCursorScreenPosition([3, 0])
@@ -446,6 +449,7 @@ describe "TextEditor", ->
describe "when line is wrapped and follow previous line indentation", ->
beforeEach ->
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(50)
it "wraps to the end of the previous line", ->
@@ -604,6 +608,7 @@ describe "TextEditor", ->
describe "when soft wrap is on", ->
it "moves cursor to the beginning of the screen line", ->
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(10)
editor.setCursorScreenPosition([1, 2])
editor.moveToEndOfScreenLine()
@@ -623,6 +628,7 @@ describe "TextEditor", ->
describe ".moveToBeginningOfLine()", ->
it "moves cursor to the beginning of the buffer line", ->
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(10)
editor.setCursorScreenPosition([1, 2])
editor.moveToBeginningOfLine()
@@ -632,6 +638,7 @@ describe "TextEditor", ->
describe ".moveToEndOfLine()", ->
it "moves cursor to the end of the buffer line", ->
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(10)
editor.setCursorScreenPosition([0, 2])
editor.moveToEndOfLine()
@@ -642,6 +649,7 @@ describe "TextEditor", ->
describe "when soft wrap is on", ->
it "moves to the first character of the current screen line or the beginning of the screen line if it's already on the first character", ->
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(10)
editor.setCursorScreenPosition [2, 5]
editor.addCursorAtScreenPosition [8, 7]
@@ -1523,6 +1531,7 @@ describe "TextEditor", ->
it "can add selections to soft-wrapped line segments", ->
editor.setSoftWrapped(true)
editor.setEditorWidthInChars(40)
editor.setDefaultCharWidth(1)
editor.setSelectedScreenRange([[3, 10], [3, 15]])
editor.addSelectionBelow()
@@ -1548,6 +1557,7 @@ describe "TextEditor", ->
describe "when lines are soft-wrapped", ->
beforeEach ->
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(40)
it "skips soft-wrap indentation tokens", ->
@@ -1633,6 +1643,7 @@ describe "TextEditor", ->
it "can add selections to soft-wrapped line segments", ->
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(40)
editor.setSelectedScreenRange([[4, 10], [4, 15]])
@@ -1659,6 +1670,7 @@ describe "TextEditor", ->
describe "when lines are soft-wrapped", ->
beforeEach ->
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(40)
it "skips soft-wrap indentation tokens", ->
@@ -2705,6 +2717,7 @@ describe "TextEditor", ->
describe "when soft wrap is on", ->
it "cuts up to the end of the line", ->
editor.setSoftWrapped(true)
editor.setDefaultCharWidth(1)
editor.setEditorWidthInChars(10)
editor.setCursorScreenPosition([2, 2])
editor.cutToEndOfLine()

30
spec/text-utils-spec.coffee
View File

@@ -44,3 +44,33 @@ describe 'text utilities', ->
expect(textUtils.isPairedCharacter('ae\u0301c', 2)).toBe false
expect(textUtils.isPairedCharacter('ae\u0301c', 3)).toBe false
expect(textUtils.isPairedCharacter('ae\u0301c', 4)).toBe false
describe ".isDoubleWidthCharacter(character)", ->
it "returns true when the character is either japanese, chinese or a full width form", ->
expect(textUtils.isDoubleWidthCharacter("我")).toBe(true)
expect(textUtils.isDoubleWidthCharacter("私")).toBe(true)
expect(textUtils.isDoubleWidthCharacter("Ｂ")).toBe(true)
expect(textUtils.isDoubleWidthCharacter("，")).toBe(true)
expect(textUtils.isDoubleWidthCharacter("￠")).toBe(true)
expect(textUtils.isDoubleWidthCharacter("a")).toBe(false)
describe ".isHalfWidthCharacter(character)", ->
it "returns true when the character is an half width form", ->
expect(textUtils.isHalfWidthCharacter("ﾊ")).toBe(true)
expect(textUtils.isHalfWidthCharacter("ﾋ")).toBe(true)
expect(textUtils.isHalfWidthCharacter("ﾬ")).toBe(true)
expect(textUtils.isHalfWidthCharacter("￭")).toBe(true)
expect(textUtils.isHalfWidthCharacter("B")).toBe(false)
describe ".isKoreanCharacter(character)", ->
it "returns true when the character is a korean character", ->
expect(textUtils.isKoreanCharacter("우")).toBe(true)
expect(textUtils.isKoreanCharacter("가")).toBe(true)
expect(textUtils.isKoreanCharacter("ㅢ")).toBe(true)
expect(textUtils.isKoreanCharacter("ㄼ")).toBe(true)
expect(textUtils.isKoreanCharacter("O")).toBe(false)

54
src/display-buffer.coffee
View File

@@ -204,10 +204,24 @@ class DisplayBuffer extends Model
getLineHeightInPixels: -> @lineHeightInPixels
setLineHeightInPixels: (@lineHeightInPixels) -> @lineHeightInPixels
getKoreanCharWidth: -> @koreanCharWidth
getHalfWidthCharWidth: -> @halfWidthCharWidth
getDoubleWidthCharWidth: -> @doubleWidthCharWidth
getDefaultCharWidth: -> @defaultCharWidth
setDefaultCharWidth: (defaultCharWidth) ->
if defaultCharWidth isnt @defaultCharWidth
setDefaultCharWidth: (defaultCharWidth, doubleWidthCharWidth, halfWidthCharWidth, koreanCharWidth) ->
doubleWidthCharWidth ?= defaultCharWidth
halfWidthCharWidth ?= defaultCharWidth
koreanCharWidth ?= defaultCharWidth
if defaultCharWidth isnt @defaultCharWidth or doubleWidthCharWidth isnt @doubleWidthCharWidth and halfWidthCharWidth isnt @halfWidthCharWidth and koreanCharWidth isnt @koreanCharWidth
@defaultCharWidth = defaultCharWidth
@doubleWidthCharWidth = doubleWidthCharWidth
@halfWidthCharWidth = halfWidthCharWidth
@koreanCharWidth = koreanCharWidth
@updateWrappedScreenLines() if @isSoftWrapped() and @getEditorWidthInChars()?
defaultCharWidth
getCursorWidth: -> 1
@@ -277,6 +291,40 @@ class DisplayBuffer extends Model
else
@getEditorWidthInChars()
getSoftWrapColumnForTokenizedLine: (tokenizedLine) ->
lineMaxWidth = @getSoftWrapColumn() * @getDefaultCharWidth()
return if Number.isNaN(lineMaxWidth)
return 0 if lineMaxWidth is 0
iterator = tokenizedLine.getTokenIterator(false)
column = 0
currentWidth = 0
while iterator.next()
textIndex = 0
text = iterator.getText()
while textIndex < text.length
if iterator.isPairedCharacter()
charLength = 2
else
charLength = 1
if iterator.hasDoubleWidthCharacterAt(textIndex)
charWidth = @getDoubleWidthCharWidth()
else if iterator.hasHalfWidthCharacterAt(textIndex)
charWidth = @getHalfWidthCharWidth()
else if iterator.hasKoreanCharacterAt(textIndex)
charWidth = @getKoreanCharWidth()
else
charWidth = @getDefaultCharWidth()
return column if currentWidth + charWidth > lineMaxWidth
currentWidth += charWidth
column += charLength
textIndex += charLength
column
# Gets the screen line for the given screen row.
#
# * `screenRow` - A {Number} indicating the screen row.
@@ -973,7 +1021,7 @@ class DisplayBuffer extends Model
else
softWraps = 0
if @isSoftWrapped()
while wrapScreenColumn = tokenizedLine.findWrapColumn(@getSoftWrapColumn())
while wrapScreenColumn = tokenizedLine.findWrapColumn(@getSoftWrapColumnForTokenizedLine(tokenizedLine))
[wrappedLine, tokenizedLine] = tokenizedLine.softWrapAt(
wrapScreenColumn,
@configSettings.softWrapHangingIndent

20
src/lines-component.coffee
View File

@@ -7,7 +7,15 @@ DummyLineNode.className = 'line'
DummyLineNode.style.position = 'absolute'
DummyLineNode.style.visibility = 'hidden'
DummyLineNode.appendChild(document.createElement('span'))
DummyLineNode.firstChild.textContent = 'x'
DummyLineNode.appendChild(document.createElement('span'))
DummyLineNode.appendChild(document.createElement('span'))
DummyLineNode.appendChild(document.createElement('span'))
DummyLineNode.children[0].textContent = 'x'
DummyLineNode.children[1].textContent = '我'
DummyLineNode.children[2].textContent = 'ﾊ'
DummyLineNode.children[3].textContent = '세'
RangeForMeasurement = document.createRange()
module.exports =
class LinesComponent extends TiledComponent
@@ -76,12 +84,18 @@ class LinesComponent extends TiledComponent
measureLineHeightAndDefaultCharWidth: ->
@domNode.appendChild(DummyLineNode)
textNode = DummyLineNode.firstChild.childNodes[0]
lineHeightInPixels = DummyLineNode.getBoundingClientRect().height
charWidth = DummyLineNode.firstChild.getBoundingClientRect().width
defaultCharWidth = DummyLineNode.children[0].getBoundingClientRect().width
doubleWidthCharWidth = DummyLineNode.children[1].getBoundingClientRect().width
halfWidthCharWidth = DummyLineNode.children[2].getBoundingClientRect().width
koreanCharWidth = DummyLineNode.children[3].getBoundingClientRect().width
@domNode.removeChild(DummyLineNode)
@presenter.setLineHeight(lineHeightInPixels)
@presenter.setBaseCharacterWidth(charWidth)
@presenter.setBaseCharacterWidth(defaultCharWidth, doubleWidthCharWidth, halfWidthCharWidth, koreanCharWidth)
lineNodeForLineIdAndScreenRow: (lineId, screenRow) ->
tile = @presenter.tileForRow(screenRow)

4
src/lines-yardstick.coffee
View File

@@ -40,7 +40,7 @@ class LinesYardstick
previousColumn = 0
previousLeft = 0
@tokenIterator.reset(line)
@tokenIterator.reset(line, false)
while @tokenIterator.next()
text = @tokenIterator.getText()
textIndex = 0
@@ -112,7 +112,7 @@ class LinesYardstick
indexWithinTextNode = null
charIndex = 0
@tokenIterator.reset(line)
@tokenIterator.reset(line, false)
while @tokenIterator.next()
break if foundIndexWithinTextNode?

9
src/text-editor-presenter.coffee
View File

@@ -1122,10 +1122,13 @@ class TextEditorPresenter
@mouseWheelScreenRow = screenRow
@didStartScrolling()
setBaseCharacterWidth: (baseCharacterWidth) ->
unless @baseCharacterWidth is baseCharacterWidth
setBaseCharacterWidth: (baseCharacterWidth, doubleWidthCharWidth, halfWidthCharWidth, koreanCharWidth) ->
unless @baseCharacterWidth is baseCharacterWidth and @doubleWidthCharWidth is doubleWidthCharWidth and @halfWidthCharWidth is halfWidthCharWidth and koreanCharWidth is @koreanCharWidth
@baseCharacterWidth = baseCharacterWidth
@model.setDefaultCharWidth(baseCharacterWidth)
@doubleWidthCharWidth = doubleWidthCharWidth
@halfWidthCharWidth = halfWidthCharWidth
@koreanCharWidth = koreanCharWidth
@model.setDefaultCharWidth(baseCharacterWidth, doubleWidthCharWidth, halfWidthCharWidth, koreanCharWidth)
@characterWidthsChanged()
characterWidthsChanged: ->

9
src/text-editor.coffee
View File

@@ -3013,8 +3013,15 @@ class TextEditor extends Model
getLineHeightInPixels: -> @displayBuffer.getLineHeightInPixels()
setLineHeightInPixels: (lineHeightInPixels) -> @displayBuffer.setLineHeightInPixels(lineHeightInPixels)
getKoreanCharWidth: -> @displayBuffer.getKoreanCharWidth()
getHalfWidthCharWidth: -> @displayBuffer.getHalfWidthCharWidth()
getDoubleWidthCharWidth: -> @displayBuffer.getDoubleWidthCharWidth()
getDefaultCharWidth: -> @displayBuffer.getDefaultCharWidth()
setDefaultCharWidth: (defaultCharWidth) -> @displayBuffer.setDefaultCharWidth(defaultCharWidth)
setDefaultCharWidth: (defaultCharWidth, doubleWidthCharWidth, halfWidthCharWidth, koreanCharWidth) ->
@displayBuffer.setDefaultCharWidth(defaultCharWidth, doubleWidthCharWidth, halfWidthCharWidth, koreanCharWidth)
setHeight: (height, reentrant=false) ->
if reentrant

34
src/text-utils.coffee
View File

@@ -57,6 +57,38 @@ isPairedCharacter = (string, index=0) ->
isVariationSequence(charCodeA, charCodeB) or
isCombinedCharacter(charCodeA, charCodeB)
isJapaneseCharacter = (charCode) ->
0x3000 <= charCode <= 0x30FF
isCjkUnifiedIdeograph = (charCode) ->
0x4E00 <= charCode <= 0x9FAF
isFullWidthForm = (charCode) ->
0xFF01 <= charCode <= 0xFF5E or
0xFFE0 <= charCode <= 0xFFE6
isDoubleWidthCharacter = (character) ->
charCode = character.charCodeAt(0)
isJapaneseCharacter(charCode) or
isCjkUnifiedIdeograph(charCode) or
isFullWidthForm(charCode)
isHalfWidthCharacter = (character) ->
charCode = character.charCodeAt(0)
0xFF65 <= charCode <= 0xFFDC or
0xFFE8 <= charCode <= 0xFFEE
isKoreanCharacter = (character) ->
charCode = character.charCodeAt(0)
0xAC00 <= charCode <= 0xD7A3 or
0x1100 <= charCode <= 0x11FF or
0x3130 <= charCode <= 0x318F or
0xA960 <= charCode <= 0xA97F or
0xD7B0 <= charCode <= 0xD7FF
# Does the given string contain at least surrogate pair, variation sequence,
# or combined character?
#
@@ -70,4 +102,4 @@ hasPairedCharacter = (string) ->
index++
false
module.exports = {isPairedCharacter, hasPairedCharacter}
module.exports = {isPairedCharacter, hasPairedCharacter, isDoubleWidthCharacter, isHalfWidthCharacter, isKoreanCharacter}

60
src/token-iterator.coffee
View File

@@ -1,19 +1,18 @@
{SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols'
{isDoubleWidthCharacter, isHalfWidthCharacter, isKoreanCharacter} = require './text-utils'
module.exports =
class TokenIterator
constructor: ({@grammarRegistry}, line) ->
@reset(line) if line?
constructor: ({@grammarRegistry}, line, enableScopes) ->
@reset(line, enableScopes) if line?
reset: (@line) ->
reset: (@line, @enableScopes=true) ->
@index = null
@bufferStart = @line.startBufferColumn
@bufferEnd = @bufferStart
@screenStart = 0
@screenEnd = 0
@scopes = @line.openScopes.map (id) => @grammarRegistry.scopeForId(id)
@scopeStarts = @scopes.slice()
@scopeEnds = []
@resetScopes() if @enableScopes
this
next: ->
@@ -21,26 +20,16 @@ class TokenIterator
if @index?
@index++
@scopeEnds.length = 0
@scopeStarts.length = 0
@bufferStart = @bufferEnd
@screenStart = @screenEnd
@clearScopeStartsAndEnds() if @enableScopes
else
@index = 0
while @index < tags.length
tag = tags[@index]
if tag < 0
scope = @grammarRegistry.scopeForId(tag)
if tag % 2 is 0
if @scopeStarts[@scopeStarts.length - 1] is scope
@scopeStarts.pop()
else
@scopeEnds.push(scope)
@scopes.pop()
else
@scopeStarts.push(scope)
@scopes.push(scope)
@handleScopeForTag(tag) if @enableScopes
@index++
else
if @isHardTab()
@@ -52,10 +41,33 @@ class TokenIterator
else
@screenEnd = @screenStart + tag
@bufferEnd = @bufferStart + tag
@text = @line.text.substring(@screenStart, @screenEnd)
return true
false
resetScopes: ->
@scopes = @line.openScopes.map (id) => @grammarRegistry.scopeForId(id)
@scopeStarts = @scopes.slice()
@scopeEnds = []
clearScopeStartsAndEnds: ->
@scopeEnds.length = 0
@scopeStarts.length = 0
handleScopeForTag: (tag) ->
scope = @grammarRegistry.scopeForId(tag)
if tag % 2 is 0
if @scopeStarts[@scopeStarts.length - 1] is scope
@scopeStarts.pop()
else
@scopeEnds.push(scope)
@scopes.pop()
else
@scopeStarts.push(scope)
@scopes.push(scope)
getBufferStart: -> @bufferStart
getBufferEnd: -> @bufferEnd
@@ -67,8 +79,7 @@ class TokenIterator
getScopes: -> @scopes
getText: ->
@line.text.substring(@screenStart, @screenEnd)
getText: -> @text
isSoftTab: ->
@line.specialTokens[@index] is SoftTab
@@ -82,5 +93,14 @@ class TokenIterator
isPairedCharacter: ->
@line.specialTokens[@index] is PairedCharacter
hasDoubleWidthCharacterAt: (charIndex) ->
isDoubleWidthCharacter(@getText()[charIndex])
hasHalfWidthCharacterAt: (charIndex) ->
isHalfWidthCharacter(@getText()[charIndex])
hasKoreanCharacterAt: (charIndex) ->
isKoreanCharacter(@getText()[charIndex])
isAtomic: ->
@isSoftTab() or @isHardTab() or @isSoftWrapIndentation() or @isPairedCharacter()

2
src/tokenized-line.coffee
View File

@@ -184,7 +184,7 @@ class TokenizedLine
@lineIsWhitespaceOnly = true
@firstTrailingWhitespaceIndex = 0
getTokenIterator: -> @tokenIterator.reset(this)
getTokenIterator: -> @tokenIterator.reset(this, arguments...)
Object.defineProperty @prototype, 'tokens', get: ->
iterator = @getTokenIterator()