diff --git a/spec/app/text-mate-grammar-spec.coffee b/spec/app/text-mate-grammar-spec.coffee index 276a96496..526717617 100644 --- a/spec/app/text-mate-grammar-spec.coffee +++ b/spec/app/text-mate-grammar-spec.coffee @@ -298,3 +298,13 @@ describe "TextMateGrammar", -> grammar = syntax.selectGrammar("style.scss") {tokens} = grammar.tokenizeLine("@mixin x() { -moz-selector: whatever; }") expect(tokens[9]).toEqual value: "-moz-selector", scopes: ["source.css.scss", "meta.property-list.scss", "meta.property-name.scss"] + + describe "when a line has more tokens than `maxTokensPerLine`", -> + it "creates a final token with the remaining text and resets the ruleStack to match the begining of the line", -> + grammar = syntax.selectGrammar("hello.js") + grammar.maxTokensPerLine = 5 + originalRuleStack = [grammar.initialRule, grammar.initialRule, grammar.initialRule] + {tokens, ruleStack} = grammar.tokenizeLine("one(two(three(four(five(_param_)))))", originalRuleStack) + expect(tokens.length).toBe 5 + expect(tokens[4].value).toBe "three(four(five(_param_)))))" + expect(ruleStack).toEqual originalRuleStack \ No newline at end of file diff --git a/src/app/text-mate-grammar.coffee b/src/app/text-mate-grammar.coffee index 93fe9559f..485fd668e 100644 --- a/src/app/text-mate-grammar.coffee +++ b/src/app/text-mate-grammar.coffee @@ -26,6 +26,7 @@ class TextMateGrammar repository: null initialRule: null firstLineRegex: null + maxTokensPerLine: 100 constructor: ({ @name, @fileTypes, @scopeName, patterns, repository, @foldingStopMarker, firstLineMatch}) -> @initialRule = new Rule(this, {@scopeName, patterns}) @@ -38,6 +39,7 @@ class TextMateGrammar @repository[name] = new Rule(this, data) tokenizeLine: (line, ruleStack=[@initialRule], firstLine=false) -> + originalRuleStack = ruleStack ruleStack = new Array(ruleStack...) # clone ruleStack tokens = [] position = 0 @@ -46,6 +48,12 @@ class TextMateGrammar previousRuleStackLength = ruleStack.length previousPosition = position + if tokens.length >= (@maxTokensPerLine - 1) + token = new Token(value: line[position..], scopes: scopes) + tokens.push token + ruleStack = originalRuleStack + break + if line.length == 0 tokens = [new Token(value: "", scopes: scopes)] return { tokens, ruleStack }