diff --git a/spec/display-buffer-spec.coffee b/spec/display-buffer-spec.coffee index 0703991de..68dd9c754 100644 --- a/spec/display-buffer-spec.coffee +++ b/spec/display-buffer-spec.coffee @@ -60,11 +60,42 @@ describe "DisplayBuffer", -> describe "soft wrapping", -> beforeEach -> - displayBuffer.setSoftWrapped(true) displayBuffer.setEditorWidthInChars(50) + displayBuffer.setSoftWrapped(true) + displayBuffer.setDefaultCharWidth(1) changeHandler.reset() describe "rendering of soft-wrapped lines", -> + describe "when there are double width characters", -> + it "takes them into account when finding the soft wrap column", -> + buffer.setText("私たちのフ是一个地方，数千名学生12345业余爱们的板作为hello world this is a pretty long latin line") + displayBuffer.setDefaultCharWidth(1, 5, 0, 0) + + expect(displayBuffer.tokenizedLineForScreenRow(0).text).toBe("私たちのフ是一个地方") + expect(displayBuffer.tokenizedLineForScreenRow(1).text).toBe("，数千名学生12345业余爱") + expect(displayBuffer.tokenizedLineForScreenRow(2).text).toBe("们的板作为hello world this is a ") + expect(displayBuffer.tokenizedLineForScreenRow(3).text).toBe("pretty long latin line") + + describe "when there are half width characters", -> + it "takes them into account when finding the soft wrap column", -> + displayBuffer.setDefaultCharWidth(1, 0, 5, 0) + buffer.setText("abcﾪﾫﾬﾈﾇﾈﾉﾊﾋﾌﾋﾌﾇﾴ￮￮￮hello world this is a pretty long line") + + expect(displayBuffer.tokenizedLineForScreenRow(0).text).toBe("abcﾪﾫﾬﾈﾇﾈﾉﾊﾋ") + expect(displayBuffer.tokenizedLineForScreenRow(1).text).toBe("ﾌﾋﾌﾇﾴ￮￮￮hello ") + expect(displayBuffer.tokenizedLineForScreenRow(2).text).toBe("world this is a pretty long line") + + describe "when there are korean characters", -> + it "takes them into account when finding the soft wrap column", -> + displayBuffer.setDefaultCharWidth(1, 0, 0, 10) + buffer.setText("1234세계를 향한 대화, 유니코 제10회유니코드국제") + + expect(displayBuffer.tokenizedLineForScreenRow(0).text).toBe("1234세계를 ") + expect(displayBuffer.tokenizedLineForScreenRow(1).text).toBe("향한 대화, ") + expect(displayBuffer.tokenizedLineForScreenRow(2).text).toBe("유니코 ") + expect(displayBuffer.tokenizedLineForScreenRow(3).text).toBe("제10회유니") + expect(displayBuffer.tokenizedLineForScreenRow(4).text).toBe("코드국제") + describe "when editor.softWrapAtPreferredLineLength is set", -> it "uses the preferred line length as the soft wrap column when it is less than the configured soft wrap column", -> atom.config.set('editor.preferredLineLength', 100) @@ -242,6 +273,7 @@ describe "DisplayBuffer", -> buffer, tabLength, editorWidthInChars: 30, config: atom.config, grammarRegistry: atom.grammars, packageManager: atom.packages, assert: -> }) + displayBuffer.setDefaultCharWidth(1) displayBuffer.setSoftWrapped(true) buffer.insert([0, 0], "the quick brown fox jumps over the lazy dog.") @@ -651,6 +683,7 @@ describe "DisplayBuffer", -> beforeEach -> tabLength = 4 + displayBuffer.setDefaultCharWidth(1) displayBuffer.setTabLength(tabLength) displayBuffer.setSoftWrapped(true) displayBuffer.setEditorWidthInChars(50) @@ -768,6 +801,7 @@ describe "DisplayBuffer", -> it "correctly translates positions on soft wrapped lines containing tabs", -> buffer.setText('\t\taa bb cc dd ee ff gg') displayBuffer.setSoftWrapped(true) + displayBuffer.setDefaultCharWidth(1) displayBuffer.setEditorWidthInChars(10) expect(displayBuffer.screenPositionForBufferPosition([0, 10], wrapAtSoftNewlines: true)).toEqual [1, 4] expect(displayBuffer.bufferPositionForScreenPosition([1, 0])).toEqual [0, 9] diff --git a/spec/text-editor-component-spec.coffee b/spec/text-editor-component-spec.coffee index 32ad147c1..b0f3ace51 100644 --- a/spec/text-editor-component-spec.coffee +++ b/spec/text-editor-component-spec.coffee @@ -2895,6 +2895,18 @@ describe "TextEditorComponent", -> expect(editor.consolidateSelections).toHaveBeenCalled() expect(event.abortKeyBinding).toHaveBeenCalled() + describe "when changing the font", -> + it "measures the default char, the korean char, the double width char and the half width char widths", -> + expect(editor.getDefaultCharWidth()).toBeCloseTo(12, 0) + + component.setFontSize(10) + nextAnimationFrame() + + expect(editor.getDefaultCharWidth()).toBeCloseTo(6, 0) + expect(editor.getKoreanCharWidth()).toBeCloseTo(9, 0) + expect(editor.getDoubleWidthCharWidth()).toBe(10) + expect(editor.getHalfWidthCharWidth()).toBe(5) + describe "hiding and showing the editor", -> describe "when the editor is hidden when it is mounted", -> it "defers measurement and rendering until the editor becomes visible", -> @@ -3212,7 +3224,9 @@ describe "TextEditorComponent", -> atom.config.set 'editor.preferredLineLength', 17, scopeSelector: '.source.coffee' atom.config.set 'editor.softWrapAtPreferredLineLength', true, scopeSelector: '.source.coffee' + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(20) + coffeeEditor.setDefaultCharWidth(1) coffeeEditor.setEditorWidthInChars(20) it "wraps lines when editor.softWrap is true for a matching scope", -> diff --git a/spec/text-editor-presenter-spec.coffee b/spec/text-editor-presenter-spec.coffee index 7688e127f..1804b7b6b 100644 --- a/spec/text-editor-presenter-spec.coffee +++ b/spec/text-editor-presenter-spec.coffee @@ -1259,6 +1259,7 @@ describe "TextEditorPresenter", -> it "only applies decorations to screen rows that are spanned by their marker when lines are soft-wrapped", -> editor.setText("a line that wraps, ok") editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(16) marker = editor.markBufferRange([[0, 0], [0, 2]]) editor.decorateMarker(marker, type: 'line', class: 'a') @@ -2244,6 +2245,7 @@ describe "TextEditorPresenter", -> it "contains states for line numbers that are visible on screen", -> editor.foldBufferRow(4) editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(50) presenter = buildPresenter(explicitHeight: 25, scrollTop: 30, lineHeight: 10, tileSize: 2) @@ -2259,6 +2261,7 @@ describe "TextEditorPresenter", -> it "updates when the editor's content changes", -> editor.foldBufferRow(4) editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(50) presenter = buildPresenter(explicitHeight: 35, scrollTop: 30, tileSize: 2) @@ -2289,6 +2292,7 @@ describe "TextEditorPresenter", -> it "correctly handles the first screen line being soft-wrapped", -> editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(30) presenter = buildPresenter(explicitHeight: 25, scrollTop: 50, tileSize: 2) @@ -2417,6 +2421,7 @@ describe "TextEditorPresenter", -> it "only applies line-number decorations to screen rows that are spanned by their marker when lines are soft-wrapped", -> editor.setText("a line that wraps, ok") editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(16) marker = editor.markBufferRange([[0, 0], [0, 2]]) editor.decorateMarker(marker, type: 'line-number', class: 'a') diff --git a/spec/text-editor-spec.coffee b/spec/text-editor-spec.coffee index 92a6435e8..8ce026741 100644 --- a/spec/text-editor-spec.coffee +++ b/spec/text-editor-spec.coffee @@ -272,6 +272,7 @@ describe "TextEditor", -> describe "when soft-wrap is enabled and code is folded", -> beforeEach -> editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(50) editor.createFold(2, 3) @@ -327,6 +328,7 @@ describe "TextEditor", -> describe "when the cursor was moved down from the beginning of an indented soft-wrapped line", -> it "moves to the beginning of the previous line", -> editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(50) editor.setCursorScreenPosition([3, 0]) @@ -379,6 +381,7 @@ describe "TextEditor", -> describe "when the cursor is at the beginning of an indented soft-wrapped line", -> it "moves to the beginning of the line's continuation on the next screen row", -> editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(50) editor.setCursorScreenPosition([3, 0]) @@ -446,6 +449,7 @@ describe "TextEditor", -> describe "when line is wrapped and follow previous line indentation", -> beforeEach -> editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(50) it "wraps to the end of the previous line", -> @@ -604,6 +608,7 @@ describe "TextEditor", -> describe "when soft wrap is on", -> it "moves cursor to the beginning of the screen line", -> editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(10) editor.setCursorScreenPosition([1, 2]) editor.moveToEndOfScreenLine() @@ -623,6 +628,7 @@ describe "TextEditor", -> describe ".moveToBeginningOfLine()", -> it "moves cursor to the beginning of the buffer line", -> editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(10) editor.setCursorScreenPosition([1, 2]) editor.moveToBeginningOfLine() @@ -632,6 +638,7 @@ describe "TextEditor", -> describe ".moveToEndOfLine()", -> it "moves cursor to the end of the buffer line", -> editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(10) editor.setCursorScreenPosition([0, 2]) editor.moveToEndOfLine() @@ -642,6 +649,7 @@ describe "TextEditor", -> describe "when soft wrap is on", -> it "moves to the first character of the current screen line or the beginning of the screen line if it's already on the first character", -> editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(10) editor.setCursorScreenPosition [2, 5] editor.addCursorAtScreenPosition [8, 7] @@ -1523,6 +1531,7 @@ describe "TextEditor", -> it "can add selections to soft-wrapped line segments", -> editor.setSoftWrapped(true) editor.setEditorWidthInChars(40) + editor.setDefaultCharWidth(1) editor.setSelectedScreenRange([[3, 10], [3, 15]]) editor.addSelectionBelow() @@ -1548,6 +1557,7 @@ describe "TextEditor", -> describe "when lines are soft-wrapped", -> beforeEach -> editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(40) it "skips soft-wrap indentation tokens", -> @@ -1633,6 +1643,7 @@ describe "TextEditor", -> it "can add selections to soft-wrapped line segments", -> editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(40) editor.setSelectedScreenRange([[4, 10], [4, 15]]) @@ -1659,6 +1670,7 @@ describe "TextEditor", -> describe "when lines are soft-wrapped", -> beforeEach -> editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(40) it "skips soft-wrap indentation tokens", -> @@ -2705,6 +2717,7 @@ describe "TextEditor", -> describe "when soft wrap is on", -> it "cuts up to the end of the line", -> editor.setSoftWrapped(true) + editor.setDefaultCharWidth(1) editor.setEditorWidthInChars(10) editor.setCursorScreenPosition([2, 2]) editor.cutToEndOfLine() diff --git a/spec/text-utils-spec.coffee b/spec/text-utils-spec.coffee index 6a03bb02f..dd528b37e 100644 --- a/spec/text-utils-spec.coffee +++ b/spec/text-utils-spec.coffee @@ -44,3 +44,33 @@ describe 'text utilities', -> expect(textUtils.isPairedCharacter('ae\u0301c', 2)).toBe false expect(textUtils.isPairedCharacter('ae\u0301c', 3)).toBe false expect(textUtils.isPairedCharacter('ae\u0301c', 4)).toBe false + + describe ".isDoubleWidthCharacter(character)", -> + it "returns true when the character is either japanese, chinese or a full width form", -> + expect(textUtils.isDoubleWidthCharacter("我")).toBe(true) + + expect(textUtils.isDoubleWidthCharacter("私")).toBe(true) + + expect(textUtils.isDoubleWidthCharacter("Ｂ")).toBe(true) + expect(textUtils.isDoubleWidthCharacter("，")).toBe(true) + expect(textUtils.isDoubleWidthCharacter("￠")).toBe(true) + + expect(textUtils.isDoubleWidthCharacter("a")).toBe(false) + + describe ".isHalfWidthCharacter(character)", -> + it "returns true when the character is an half width form", -> + expect(textUtils.isHalfWidthCharacter("ﾊ")).toBe(true) + expect(textUtils.isHalfWidthCharacter("ﾋ")).toBe(true) + expect(textUtils.isHalfWidthCharacter("ﾬ")).toBe(true) + expect(textUtils.isHalfWidthCharacter("￭")).toBe(true) + + expect(textUtils.isHalfWidthCharacter("B")).toBe(false) + + describe ".isKoreanCharacter(character)", -> + it "returns true when the character is a korean character", -> + expect(textUtils.isKoreanCharacter("우")).toBe(true) + expect(textUtils.isKoreanCharacter("가")).toBe(true) + expect(textUtils.isKoreanCharacter("ㅢ")).toBe(true) + expect(textUtils.isKoreanCharacter("ㄼ")).toBe(true) + + expect(textUtils.isKoreanCharacter("O")).toBe(false) diff --git a/src/display-buffer.coffee b/src/display-buffer.coffee index 8d8fa84cc..1021955cf 100644 --- a/src/display-buffer.coffee +++ b/src/display-buffer.coffee @@ -204,10 +204,24 @@ class DisplayBuffer extends Model getLineHeightInPixels: -> @lineHeightInPixels setLineHeightInPixels: (@lineHeightInPixels) -> @lineHeightInPixels + getKoreanCharWidth: -> @koreanCharWidth + + getHalfWidthCharWidth: -> @halfWidthCharWidth + + getDoubleWidthCharWidth: -> @doubleWidthCharWidth + getDefaultCharWidth: -> @defaultCharWidth - setDefaultCharWidth: (defaultCharWidth) -> - if defaultCharWidth isnt @defaultCharWidth + + setDefaultCharWidth: (defaultCharWidth, doubleWidthCharWidth, halfWidthCharWidth, koreanCharWidth) -> + doubleWidthCharWidth ?= defaultCharWidth + halfWidthCharWidth ?= defaultCharWidth + koreanCharWidth ?= defaultCharWidth + if defaultCharWidth isnt @defaultCharWidth or doubleWidthCharWidth isnt @doubleWidthCharWidth and halfWidthCharWidth isnt @halfWidthCharWidth and koreanCharWidth isnt @koreanCharWidth @defaultCharWidth = defaultCharWidth + @doubleWidthCharWidth = doubleWidthCharWidth + @halfWidthCharWidth = halfWidthCharWidth + @koreanCharWidth = koreanCharWidth + @updateWrappedScreenLines() if @isSoftWrapped() and @getEditorWidthInChars()? defaultCharWidth getCursorWidth: -> 1 @@ -277,6 +291,40 @@ class DisplayBuffer extends Model else @getEditorWidthInChars() + getSoftWrapColumnForTokenizedLine: (tokenizedLine) -> + lineMaxWidth = @getSoftWrapColumn() * @getDefaultCharWidth() + + return if Number.isNaN(lineMaxWidth) + return 0 if lineMaxWidth is 0 + + iterator = tokenizedLine.getTokenIterator(false) + column = 0 + currentWidth = 0 + while iterator.next() + textIndex = 0 + text = iterator.getText() + while textIndex < text.length + if iterator.isPairedCharacter() + charLength = 2 + else + charLength = 1 + + if iterator.hasDoubleWidthCharacterAt(textIndex) + charWidth = @getDoubleWidthCharWidth() + else if iterator.hasHalfWidthCharacterAt(textIndex) + charWidth = @getHalfWidthCharWidth() + else if iterator.hasKoreanCharacterAt(textIndex) + charWidth = @getKoreanCharWidth() + else + charWidth = @getDefaultCharWidth() + + return column if currentWidth + charWidth > lineMaxWidth + + currentWidth += charWidth + column += charLength + textIndex += charLength + column + # Gets the screen line for the given screen row. # # * `screenRow` - A {Number} indicating the screen row. @@ -973,7 +1021,7 @@ class DisplayBuffer extends Model else softWraps = 0 if @isSoftWrapped() - while wrapScreenColumn = tokenizedLine.findWrapColumn(@getSoftWrapColumn()) + while wrapScreenColumn = tokenizedLine.findWrapColumn(@getSoftWrapColumnForTokenizedLine(tokenizedLine)) [wrappedLine, tokenizedLine] = tokenizedLine.softWrapAt( wrapScreenColumn, @configSettings.softWrapHangingIndent diff --git a/src/lines-component.coffee b/src/lines-component.coffee index 3ae555f3c..b5af56885 100644 --- a/src/lines-component.coffee +++ b/src/lines-component.coffee @@ -7,7 +7,15 @@ DummyLineNode.className = 'line' DummyLineNode.style.position = 'absolute' DummyLineNode.style.visibility = 'hidden' DummyLineNode.appendChild(document.createElement('span')) -DummyLineNode.firstChild.textContent = 'x' +DummyLineNode.appendChild(document.createElement('span')) +DummyLineNode.appendChild(document.createElement('span')) +DummyLineNode.appendChild(document.createElement('span')) +DummyLineNode.children[0].textContent = 'x' +DummyLineNode.children[1].textContent = '我' +DummyLineNode.children[2].textContent = 'ﾊ' +DummyLineNode.children[3].textContent = '세' + +RangeForMeasurement = document.createRange() module.exports = class LinesComponent extends TiledComponent @@ -76,12 +84,18 @@ class LinesComponent extends TiledComponent measureLineHeightAndDefaultCharWidth: -> @domNode.appendChild(DummyLineNode) + textNode = DummyLineNode.firstChild.childNodes[0] + lineHeightInPixels = DummyLineNode.getBoundingClientRect().height - charWidth = DummyLineNode.firstChild.getBoundingClientRect().width + defaultCharWidth = DummyLineNode.children[0].getBoundingClientRect().width + doubleWidthCharWidth = DummyLineNode.children[1].getBoundingClientRect().width + halfWidthCharWidth = DummyLineNode.children[2].getBoundingClientRect().width + koreanCharWidth = DummyLineNode.children[3].getBoundingClientRect().width + @domNode.removeChild(DummyLineNode) @presenter.setLineHeight(lineHeightInPixels) - @presenter.setBaseCharacterWidth(charWidth) + @presenter.setBaseCharacterWidth(defaultCharWidth, doubleWidthCharWidth, halfWidthCharWidth, koreanCharWidth) lineNodeForLineIdAndScreenRow: (lineId, screenRow) -> tile = @presenter.tileForRow(screenRow) diff --git a/src/lines-yardstick.coffee b/src/lines-yardstick.coffee index 3254451f1..fa00bae40 100644 --- a/src/lines-yardstick.coffee +++ b/src/lines-yardstick.coffee @@ -40,7 +40,7 @@ class LinesYardstick previousColumn = 0 previousLeft = 0 - @tokenIterator.reset(line) + @tokenIterator.reset(line, false) while @tokenIterator.next() text = @tokenIterator.getText() textIndex = 0 @@ -112,7 +112,7 @@ class LinesYardstick indexWithinTextNode = null charIndex = 0 - @tokenIterator.reset(line) + @tokenIterator.reset(line, false) while @tokenIterator.next() break if foundIndexWithinTextNode? diff --git a/src/text-editor-presenter.coffee b/src/text-editor-presenter.coffee index d75bd37c3..f416f171c 100644 --- a/src/text-editor-presenter.coffee +++ b/src/text-editor-presenter.coffee @@ -1122,10 +1122,13 @@ class TextEditorPresenter @mouseWheelScreenRow = screenRow @didStartScrolling() - setBaseCharacterWidth: (baseCharacterWidth) -> - unless @baseCharacterWidth is baseCharacterWidth + setBaseCharacterWidth: (baseCharacterWidth, doubleWidthCharWidth, halfWidthCharWidth, koreanCharWidth) -> + unless @baseCharacterWidth is baseCharacterWidth and @doubleWidthCharWidth is doubleWidthCharWidth and @halfWidthCharWidth is halfWidthCharWidth and koreanCharWidth is @koreanCharWidth @baseCharacterWidth = baseCharacterWidth - @model.setDefaultCharWidth(baseCharacterWidth) + @doubleWidthCharWidth = doubleWidthCharWidth + @halfWidthCharWidth = halfWidthCharWidth + @koreanCharWidth = koreanCharWidth + @model.setDefaultCharWidth(baseCharacterWidth, doubleWidthCharWidth, halfWidthCharWidth, koreanCharWidth) @characterWidthsChanged() characterWidthsChanged: -> diff --git a/src/text-editor.coffee b/src/text-editor.coffee index 5f5610dec..4c6a0f514 100644 --- a/src/text-editor.coffee +++ b/src/text-editor.coffee @@ -3013,8 +3013,15 @@ class TextEditor extends Model getLineHeightInPixels: -> @displayBuffer.getLineHeightInPixels() setLineHeightInPixels: (lineHeightInPixels) -> @displayBuffer.setLineHeightInPixels(lineHeightInPixels) + getKoreanCharWidth: -> @displayBuffer.getKoreanCharWidth() + + getHalfWidthCharWidth: -> @displayBuffer.getHalfWidthCharWidth() + + getDoubleWidthCharWidth: -> @displayBuffer.getDoubleWidthCharWidth() + getDefaultCharWidth: -> @displayBuffer.getDefaultCharWidth() - setDefaultCharWidth: (defaultCharWidth) -> @displayBuffer.setDefaultCharWidth(defaultCharWidth) + setDefaultCharWidth: (defaultCharWidth, doubleWidthCharWidth, halfWidthCharWidth, koreanCharWidth) -> + @displayBuffer.setDefaultCharWidth(defaultCharWidth, doubleWidthCharWidth, halfWidthCharWidth, koreanCharWidth) setHeight: (height, reentrant=false) -> if reentrant diff --git a/src/text-utils.coffee b/src/text-utils.coffee index 37955dcd6..82bed4da5 100644 --- a/src/text-utils.coffee +++ b/src/text-utils.coffee @@ -57,6 +57,38 @@ isPairedCharacter = (string, index=0) -> isVariationSequence(charCodeA, charCodeB) or isCombinedCharacter(charCodeA, charCodeB) +isJapaneseCharacter = (charCode) -> + 0x3000 <= charCode <= 0x30FF + +isCjkUnifiedIdeograph = (charCode) -> + 0x4E00 <= charCode <= 0x9FAF + +isFullWidthForm = (charCode) -> + 0xFF01 <= charCode <= 0xFF5E or + 0xFFE0 <= charCode <= 0xFFE6 + +isDoubleWidthCharacter = (character) -> + charCode = character.charCodeAt(0) + + isJapaneseCharacter(charCode) or + isCjkUnifiedIdeograph(charCode) or + isFullWidthForm(charCode) + +isHalfWidthCharacter = (character) -> + charCode = character.charCodeAt(0) + + 0xFF65 <= charCode <= 0xFFDC or + 0xFFE8 <= charCode <= 0xFFEE + +isKoreanCharacter = (character) -> + charCode = character.charCodeAt(0) + + 0xAC00 <= charCode <= 0xD7A3 or + 0x1100 <= charCode <= 0x11FF or + 0x3130 <= charCode <= 0x318F or + 0xA960 <= charCode <= 0xA97F or + 0xD7B0 <= charCode <= 0xD7FF + # Does the given string contain at least surrogate pair, variation sequence, # or combined character? # @@ -70,4 +102,4 @@ hasPairedCharacter = (string) -> index++ false -module.exports = {isPairedCharacter, hasPairedCharacter} +module.exports = {isPairedCharacter, hasPairedCharacter, isDoubleWidthCharacter, isHalfWidthCharacter, isKoreanCharacter} diff --git a/src/token-iterator.coffee b/src/token-iterator.coffee index cc31ca98e..8f0fe202f 100644 --- a/src/token-iterator.coffee +++ b/src/token-iterator.coffee @@ -1,19 +1,18 @@ {SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols' +{isDoubleWidthCharacter, isHalfWidthCharacter, isKoreanCharacter} = require './text-utils' module.exports = class TokenIterator - constructor: ({@grammarRegistry}, line) -> - @reset(line) if line? + constructor: ({@grammarRegistry}, line, enableScopes) -> + @reset(line, enableScopes) if line? - reset: (@line) -> + reset: (@line, @enableScopes=true) -> @index = null @bufferStart = @line.startBufferColumn @bufferEnd = @bufferStart @screenStart = 0 @screenEnd = 0 - @scopes = @line.openScopes.map (id) => @grammarRegistry.scopeForId(id) - @scopeStarts = @scopes.slice() - @scopeEnds = [] + @resetScopes() if @enableScopes this next: -> @@ -21,26 +20,16 @@ class TokenIterator if @index? @index++ - @scopeEnds.length = 0 - @scopeStarts.length = 0 @bufferStart = @bufferEnd @screenStart = @screenEnd + @clearScopeStartsAndEnds() if @enableScopes else @index = 0 while @index < tags.length tag = tags[@index] if tag < 0 - scope = @grammarRegistry.scopeForId(tag) - if tag % 2 is 0 - if @scopeStarts[@scopeStarts.length - 1] is scope - @scopeStarts.pop() - else - @scopeEnds.push(scope) - @scopes.pop() - else - @scopeStarts.push(scope) - @scopes.push(scope) + @handleScopeForTag(tag) if @enableScopes @index++ else if @isHardTab() @@ -52,10 +41,33 @@ class TokenIterator else @screenEnd = @screenStart + tag @bufferEnd = @bufferStart + tag + + @text = @line.text.substring(@screenStart, @screenEnd) return true false + resetScopes: -> + @scopes = @line.openScopes.map (id) => @grammarRegistry.scopeForId(id) + @scopeStarts = @scopes.slice() + @scopeEnds = [] + + clearScopeStartsAndEnds: -> + @scopeEnds.length = 0 + @scopeStarts.length = 0 + + handleScopeForTag: (tag) -> + scope = @grammarRegistry.scopeForId(tag) + if tag % 2 is 0 + if @scopeStarts[@scopeStarts.length - 1] is scope + @scopeStarts.pop() + else + @scopeEnds.push(scope) + @scopes.pop() + else + @scopeStarts.push(scope) + @scopes.push(scope) + getBufferStart: -> @bufferStart getBufferEnd: -> @bufferEnd @@ -67,8 +79,7 @@ class TokenIterator getScopes: -> @scopes - getText: -> - @line.text.substring(@screenStart, @screenEnd) + getText: -> @text isSoftTab: -> @line.specialTokens[@index] is SoftTab @@ -82,5 +93,14 @@ class TokenIterator isPairedCharacter: -> @line.specialTokens[@index] is PairedCharacter + hasDoubleWidthCharacterAt: (charIndex) -> + isDoubleWidthCharacter(@getText()[charIndex]) + + hasHalfWidthCharacterAt: (charIndex) -> + isHalfWidthCharacter(@getText()[charIndex]) + + hasKoreanCharacterAt: (charIndex) -> + isKoreanCharacter(@getText()[charIndex]) + isAtomic: -> @isSoftTab() or @isHardTab() or @isSoftWrapIndentation() or @isPairedCharacter() diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index bf234b6d3..2a27d3f12 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -184,7 +184,7 @@ class TokenizedLine @lineIsWhitespaceOnly = true @firstTrailingWhitespaceIndex = 0 - getTokenIterator: -> @tokenIterator.reset(this) + getTokenIterator: -> @tokenIterator.reset(this, arguments...) Object.defineProperty @prototype, 'tokens', get: -> iterator = @getTokenIterator()