From c56ac701812d98fa7ce23cfc4c2d263a0928eb5e Mon Sep 17 00:00:00 2001 From: probablycorey Date: Wed, 28 May 2014 14:53:38 -0700 Subject: [PATCH 1/6] Add `tokenized` event to tokenized buffer --- spec/tokenized-buffer-spec.coffee | 14 ++++++++++++++ src/tokenized-buffer.coffee | 5 ++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee index c1f29196a..68a8cf54c 100644 --- a/spec/tokenized-buffer-spec.coffee +++ b/spec/tokenized-buffer-spec.coffee @@ -385,6 +385,20 @@ describe "TokenizedBuffer", -> expect(tokens[1].value).toBeTruthy() expect(tokens[2].value).toBe 'xyz' + describe "when the grammar tokenized", -> + it "emits the `tokenized` event", -> + buffer = null + tokenizedBuffer = null + tokenizedHandler = jasmine.createSpy("tokenized handler") + + waitsForPromise -> + atom.project.open('sample.js').then (editor) -> buffer = editor.getBuffer() + + runs -> + tokenizedBuffer = new TokenizedBuffer({buffer}) + tokenizedBuffer.on 'tokenized', tokenizedHandler + fullyTokenize(tokenizedBuffer) + expect(tokenizedHandler.callCount).toBe(1) describe "when the grammar is updated because a grammar it includes is activated", -> it "retokenizes the buffer", -> diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index 01886ee0a..1d17249e2 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -124,7 +124,10 @@ class TokenizedBuffer extends Model @invalidateRow(row + 1) unless filledRegion @emit "changed", { start: invalidRow, end: row, delta: 0 } - @tokenizeInBackground() if @firstInvalidRow()? + if @firstInvalidRow()? + @tokenizeInBackground() + else + @emit "tokenized" firstInvalidRow: -> @invalidRows[0] From 2c60b0463e50b13b322b23b81a7c9a323132dc04 Mon Sep 17 00:00:00 2001 From: probablycorey Date: Wed, 28 May 2014 15:46:26 -0700 Subject: [PATCH 2/6] Use tokenized buffer created by editor --- spec/tokenized-buffer-spec.coffee | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee index 68a8cf54c..3ada12205 100644 --- a/spec/tokenized-buffer-spec.coffee +++ b/spec/tokenized-buffer-spec.coffee @@ -387,6 +387,17 @@ describe "TokenizedBuffer", -> describe "when the grammar tokenized", -> it "emits the `tokenized` event", -> + editor = null + tokenizedHandler = jasmine.createSpy("tokenized handler") + + waitsForPromise -> + atom.project.open('sample.js').then (o) -> editor = o + + runs -> + tokenizedBuffer = editor.displayBuffer.tokenizedBuffer + tokenizedBuffer.on 'tokenized', tokenizedHandler + fullyTokenize(tokenizedBuffer) + expect(tokenizedHandler.callCount).toBe(1) buffer = null tokenizedBuffer = null tokenizedHandler = jasmine.createSpy("tokenized handler") From 89dc5f26add13aae6506896c450cb31f737e613d Mon Sep 17 00:00:00 2001 From: probablycorey Date: Wed, 28 May 2014 15:56:08 -0700 Subject: [PATCH 3/6] Only emit the tokenized event after the first full tokenization --- spec/tokenized-buffer-spec.coffee | 16 ++++++++++++++++ src/tokenized-buffer.coffee | 4 +++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee index 3ada12205..413bf4802 100644 --- a/spec/tokenized-buffer-spec.coffee +++ b/spec/tokenized-buffer-spec.coffee @@ -398,6 +398,22 @@ describe "TokenizedBuffer", -> tokenizedBuffer.on 'tokenized', tokenizedHandler fullyTokenize(tokenizedBuffer) expect(tokenizedHandler.callCount).toBe(1) + + it "doesn't re-emit the `tokenized` event when a line is edited", -> + editor = null + tokenizedHandler = jasmine.createSpy("tokenized handler") + + waitsForPromise -> + atom.project.open('sample.js').then (o) -> editor = o + + runs -> + tokenizedBuffer = editor.displayBuffer.tokenizedBuffer + fullyTokenize(tokenizedBuffer) + + tokenizedBuffer.on 'tokenized', tokenizedHandler + editor.getBuffer().insert([0, 0], "'") + fullyTokenize(tokenizedBuffer) + expect(tokenizedHandler).not.toHaveBeenCalled() buffer = null tokenizedBuffer = null tokenizedHandler = jasmine.createSpy("tokenized handler") diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index 1d17249e2..97afcd89b 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -78,6 +78,7 @@ class TokenizedBuffer extends Model @tokenizedLines = @buildPlaceholderTokenizedLinesForRows(0, @buffer.getLastRow()) @invalidRows = [] @invalidateRow(0) + @fullyTokenized = false setVisible: (@visible) -> @tokenizeInBackground() if @visible @@ -127,7 +128,8 @@ class TokenizedBuffer extends Model if @firstInvalidRow()? @tokenizeInBackground() else - @emit "tokenized" + @emit "tokenized" unless @fullyTokenized + @fullyTokenized = true firstInvalidRow: -> @invalidRows[0] From b7df08cbdd87a6bdb2221decd751a70adb0d412e Mon Sep 17 00:00:00 2001 From: probablycorey Date: Wed, 28 May 2014 16:09:42 -0700 Subject: [PATCH 4/6] Add spec to re-emit the tokenized event when the grammar is changed --- spec/tokenized-buffer-spec.coffee | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee index 413bf4802..c7ae73bcf 100644 --- a/spec/tokenized-buffer-spec.coffee +++ b/spec/tokenized-buffer-spec.coffee @@ -414,19 +414,29 @@ describe "TokenizedBuffer", -> editor.getBuffer().insert([0, 0], "'") fullyTokenize(tokenizedBuffer) expect(tokenizedHandler).not.toHaveBeenCalled() - buffer = null + + describe "when the grammar is updated because a grammar it includes is activated", -> + it "re-emits the `tokenized` event", -> + editor = null tokenizedBuffer = null tokenizedHandler = jasmine.createSpy("tokenized handler") waitsForPromise -> - atom.project.open('sample.js').then (editor) -> buffer = editor.getBuffer() + atom.project.open('coffee.coffee').then (o) -> editor = o runs -> - tokenizedBuffer = new TokenizedBuffer({buffer}) + tokenizedBuffer = editor.displayBuffer.tokenizedBuffer tokenizedBuffer.on 'tokenized', tokenizedHandler + fullyTokenize(tokenizedBuffer) + tokenizedHandler.reset() + + waitsForPromise -> + atom.packages.activatePackage('language-coffee-script') + + runs -> fullyTokenize(tokenizedBuffer) expect(tokenizedHandler.callCount).toBe(1) - describe "when the grammar is updated because a grammar it includes is activated", -> + it "retokenizes the buffer", -> waitsForPromise -> From 7f57a094f6c4f470c9223b1706f70cc8b7953075 Mon Sep 17 00:00:00 2001 From: probablycorey Date: Wed, 28 May 2014 16:31:00 -0700 Subject: [PATCH 5/6] Determine softTab state after the buffer is tokenized. --- spec/editor-spec.coffee | 12 ++++++++++++ .../sample-with-tabs-and-leading-comment.coffee | 4 ++++ src/display-buffer.coffee | 1 + src/editor.coffee | 4 ++++ 4 files changed, 21 insertions(+) create mode 100644 spec/fixtures/sample-with-tabs-and-leading-comment.coffee diff --git a/spec/editor-spec.coffee b/spec/editor-spec.coffee index f037ac6af..4a2cce174 100644 --- a/spec/editor-spec.coffee +++ b/spec/editor-spec.coffee @@ -2876,6 +2876,18 @@ describe "Editor", -> expect(editor.lineForBufferRow(4)).toBe " }" expect(editor.lineForBufferRow(5)).toBe " i=1" + describe "soft and hard tabs", -> + it "resets the tab style when tokenization is complete", -> + editor.destroy() + atom.project.open('sample-with-tabs-and-leading-comment.coffee').then (o) -> editor = o + expect(editor.softTabs).toBe true + + waitsForPromise -> + atom.packages.activatePackage('language-coffee-script') + + runs -> + expect(editor.softTabs).toBe false + describe ".destroy()", -> it "destroys all markers associated with the edit session", -> expect(buffer.getMarkerCount()).toBeGreaterThan 0 diff --git a/spec/fixtures/sample-with-tabs-and-leading-comment.coffee b/spec/fixtures/sample-with-tabs-and-leading-comment.coffee new file mode 100644 index 000000000..0f81f6fe8 --- /dev/null +++ b/spec/fixtures/sample-with-tabs-and-leading-comment.coffee @@ -0,0 +1,4 @@ + # This is a comment +if this.studyingEconomics + buy() while supply > demand + sell() until supply > demand diff --git a/src/display-buffer.coffee b/src/display-buffer.coffee index 1c753e24d..579863ec2 100644 --- a/src/display-buffer.coffee +++ b/src/display-buffer.coffee @@ -46,6 +46,7 @@ class DisplayBuffer extends Model @updateAllScreenLines() @createFoldForMarker(marker) for marker in @buffer.findMarkers(@getFoldMarkerAttributes()) @subscribe @tokenizedBuffer, 'grammar-changed', (grammar) => @emit 'grammar-changed', grammar + @subscribe @tokenizedBuffer, 'tokenized', => @emit 'tokenized' @subscribe @tokenizedBuffer, 'changed', @handleTokenizedBufferChange @subscribe @buffer, 'markers-updated', @handleBufferMarkersUpdated @subscribe @buffer, 'marker-created', @handleBufferMarkerCreated diff --git a/src/editor.coffee b/src/editor.coffee index 161a4455c..5f7ff13eb 100644 --- a/src/editor.coffee +++ b/src/editor.coffee @@ -211,6 +211,7 @@ class Editor extends Model @subscribe @displayBuffer, "changed", (e) => @emit 'screen-lines-changed', e @subscribe @displayBuffer, "markers-updated", => @mergeIntersectingSelections() @subscribe @displayBuffer, 'grammar-changed', => @handleGrammarChange() + @subscribe @displayBuffer, 'tokenized', => @handleTokenization() @subscribe @displayBuffer, 'soft-wrap-changed', (args...) => @emit 'soft-wrap-changed', args... getViewClass: -> @@ -1850,6 +1851,9 @@ class Editor extends Model logScreenLines: (start, end) -> @displayBuffer.logLines(start, end) + handleTokenization: -> + @softTabs = @usesSoftTabs() ? @softTabs + handleGrammarChange: -> @unfoldAll() @emit 'grammar-changed' From c9ae9e11c11e0209bcfc77d00a44054757fba9ea Mon Sep 17 00:00:00 2001 From: probablycorey Date: Wed, 28 May 2014 16:34:42 -0700 Subject: [PATCH 6/6] Reword specs --- spec/tokenized-buffer-spec.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee index c7ae73bcf..d21653a63 100644 --- a/spec/tokenized-buffer-spec.coffee +++ b/spec/tokenized-buffer-spec.coffee @@ -385,7 +385,7 @@ describe "TokenizedBuffer", -> expect(tokens[1].value).toBeTruthy() expect(tokens[2].value).toBe 'xyz' - describe "when the grammar tokenized", -> + describe "when the grammar is tokenized", -> it "emits the `tokenized` event", -> editor = null tokenizedHandler = jasmine.createSpy("tokenized handler") @@ -399,7 +399,7 @@ describe "TokenizedBuffer", -> fullyTokenize(tokenizedBuffer) expect(tokenizedHandler.callCount).toBe(1) - it "doesn't re-emit the `tokenized` event when a line is edited", -> + it "doesn't re-emit the `tokenized` event when it is re-tokenized", -> editor = null tokenizedHandler = jasmine.createSpy("tokenized handler")