Associate TokenizedLines with an ::indentLevel

This can be used to render the appropriate number of indent guide spans
for empty lines.

spec/tokenized-buffer-spec.coffee

@@ -509,3 +509,31 @@ describe "TokenizedBuffer", ->
       expect(segment1.tokens[5].hasTrailingWhitespace).toBe false
       expect(segment2.tokens[6].value).toBe '  '
       expect(segment2.tokens[6].hasTrailingWhitespace).toBe true
+
+  describe "indent level", ->
+    beforeEach ->
+      buffer = atom.project.bufferForPathSync('sample.js')
+      tokenizedBuffer = new TokenizedBuffer({buffer})
+      fullyTokenize(tokenizedBuffer)
+
+    describe "when the line is non-empty", ->
+      it "has an indent level based on the leading whitespace on the line", ->
+        expect(tokenizedBuffer.lineForScreenRow(0).indentLevel).toBe 0
+        expect(tokenizedBuffer.lineForScreenRow(1).indentLevel).toBe 1
+        expect(tokenizedBuffer.lineForScreenRow(2).indentLevel).toBe 2
+        buffer.insert([2, 0], ' ')
+        expect(tokenizedBuffer.lineForScreenRow(2).indentLevel).toBe 2.5
+
+    describe "when the line is empty", ->
+      it "assumes the indentation level of the first non-empty line below or above if one exists", ->
+        buffer.insert([12, 0], '    ')
+        buffer.insert([12, Infinity], '\n\n')
+        expect(tokenizedBuffer.lineForScreenRow(13).indentLevel).toBe 2
+        expect(tokenizedBuffer.lineForScreenRow(14).indentLevel).toBe 2
+
+        buffer.insert([1, Infinity], '\n\n')
+        expect(tokenizedBuffer.lineForScreenRow(2).indentLevel).toBe 2
+        expect(tokenizedBuffer.lineForScreenRow(3).indentLevel).toBe 2
+
+        buffer.setText('\n\n\n')
+        expect(tokenizedBuffer.lineForScreenRow(1).indentLevel).toBe 0

src/display-buffer.coffee

@@ -242,6 +242,9 @@ class DisplayBuffer extends Model
   getLines: ->
     new Array(@screenLines...)
 
+  indentLevelForLine: (line) ->
+    @tokenizedBuffer.indentLevelForLine(line)
+
   # Given starting and ending screen rows, this returns an array of the
   # buffer rows corresponding to every screen row in the range
   #

src/editor.coffee

@@ -405,13 +405,7 @@ class Editor extends Model
   #
   # Returns a {Number}.
   indentLevelForLine: (line) ->
-    if match = line.match(/^[\t ]+/)
-      leadingWhitespace = match[0]
-      tabCount = leadingWhitespace.match(/\t/g)?.length ? 0
-      spaceCount = leadingWhitespace.match(/[ ]/g)?.length ? 0
-      tabCount + (spaceCount / @getTabLength())
-    else
-      0
+    @displayBuffer.indentLevelForLine(line)
 
   # Constructs the string used for tabs.
   buildIndentString: (number) ->

src/tokenized-buffer.coffee

@@ -185,14 +185,16 @@ class TokenizedBuffer extends Model
     line = @buffer.lineForRow(row)
     tokens = [new Token(value: line, scopes: [@grammar.scopeName])]
     tabLength = @getTabLength()
-    new TokenizedLine({tokens, tabLength})
+    indentLevel = @indentLevelForRow(row)
+    new TokenizedLine({tokens, tabLength, indentLevel})
 
   buildTokenizedTokenizedLineForRow: (row, ruleStack) ->
     line = @buffer.lineForRow(row)
     lineEnding = @buffer.lineEndingForRow(row)
     tabLength = @getTabLength()
+    indentLevel = @indentLevelForRow(row)
     { tokens, ruleStack } = @grammar.tokenizeLine(line, ruleStack, row is 0)
-    new TokenizedLine({tokens, ruleStack, tabLength, lineEnding})
+    new TokenizedLine({tokens, ruleStack, tabLength, lineEnding, indentLevel})
 
   # FIXME: benogle says: These are actually buffer rows as all buffer rows are
   # accounted for in @tokenizedLines
@@ -207,6 +209,36 @@ class TokenizedBuffer extends Model
   stackForRow: (row) ->
     @tokenizedLines[row]?.ruleStack
 
+  indentLevelForRow: (row) ->
+    line = @buffer.lineForRow(row)
+
+    if line is ''
+      nextRow = row + 1
+      lineCount = @getLineCount()
+      while nextRow < lineCount
+        nextLine = @buffer.lineForRow(nextRow)
+        return @indentLevelForLine(nextLine) unless nextLine is ''
+        nextRow++
+
+      previousRow = row - 1
+      while previousRow >= 0
+        previousLine = @buffer.lineForRow(previousRow)
+        return @indentLevelForLine(previousLine) unless previousLine is ''
+        previousRow--
+
+      0
+    else
+      @indentLevelForLine(line)
+
+  indentLevelForLine: (line) ->
+    if match = line.match(/^[\t ]+/)
+      leadingWhitespace = match[0]
+      tabCount = leadingWhitespace.match(/\t/g)?.length ? 0
+      spaceCount = leadingWhitespace.match(/[ ]/g)?.length ? 0
+      tabCount + (spaceCount / @getTabLength())
+    else
+      0
+
   scopesForPosition: (position) ->
     @tokenForPosition(position).scopes
 
@@ -306,6 +338,9 @@ class TokenizedBuffer extends Model
   getLastRow: ->
     @buffer.getLastRow()
 
+  getLineCount: ->
+    @buffer.getLineCount()
+
   logLines: (start=0, end=@buffer.getLastRow()) ->
     for row in [start..end]
       line = @lineForScreenRow(row).text

src/tokenized-line.coffee

@@ -4,7 +4,7 @@ idCounter = 1
 
 module.exports =
 class TokenizedLine
-  constructor: ({tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold, tabLength}) ->
+  constructor: ({tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold, tabLength, @indentLevel}) ->
     @tokens = @breakOutAtomicTokens(tokens, tabLength)
     @startBufferColumn ?= 0
     @text = _.pluck(@tokens, 'value').join('')