Merge pull request #885 from atom/cj-project-scan

Project scan changes for new searching behavior

package.json

@@ -22,7 +22,6 @@
     "mkdirp": "0.3.5",
     "less": "git://github.com/nathansobo/less.js.git",
     "less-cache": "0.8.0",
-    "nak": "0.2.18",
     "nslog": "0.1.0",
     "oniguruma": "0.20.0",
     "optimist": "0.4.0",
@@ -30,6 +29,7 @@
     "pegjs": "0.7.0",
     "plist": "git://github.com/nathansobo/node-plist.git",
     "rimraf": "2.1.4",
+    "scandal": "0.2.0",
     "season": "0.13.0",
     "semver": "1.1.4",
     "space-pen": "1.2.0",
@@ -38,9 +38,9 @@
     "temp": "0.5.0",
     "underscore": "1.4.4",
 
-    "atom-light-ui": "0.2.1",
+    "atom-light-ui": "0.3.0",
     "atom-light-syntax": "0.2.0",
-    "atom-dark-ui": "0.2.0",
+    "atom-dark-ui": "0.3.0",
     "atom-dark-syntax": "0.2.0",
     "base16-tomorrow-dark-theme": "0.1.0",
     "solarized-dark-syntax": "0.1.0",
@@ -55,7 +55,7 @@
     "command-palette": "0.3.0",
     "editor-stats": "0.2.0",
     "exception-reporting": "0.1.0",
-    "find-and-replace": "0.11.0",
+    "find-and-replace": "0.13.0",
     "fuzzy-finder": "0.5.0",
     "gfm": "0.4.0",
     "git-diff": "0.3.0",
@@ -117,6 +117,7 @@
   },
   "devDependencies": {
     "biscotto": "0.0.17",
+    "fstream": "0.1.24",
     "grunt": "~0.4.1",
     "grunt-cli": "~0.1.9",
     "grunt-coffeelint": "0.0.6",

spec/fixtures/git/working-dir/.gitignore

@@ -1 +1,2 @@
-ignored.txt
+poop
+ignored.txt

spec/fixtures/git/working-dir/git.git/HEAD

@@ -0,0 +1 @@
+ref: refs/heads/master

spec/fixtures/git/working-dir/git.git/config

@@ -0,0 +1,6 @@
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = false
+	logallrefupdates = true
+	ignorecase = true

spec/fixtures/git/working-dir/git.git/refs/heads/master

@@ -0,0 +1 @@
+ef046e9eecaa5255ea5e9817132d4001724d6ae1

spec/git-spec.coffee

@@ -1,3 +1,4 @@
+temp = require 'temp'
 Git = require '../src/git'
 {fs} = require 'atom'
 path = require 'path'

spec/project-spec.coffee

@@ -1,3 +1,5 @@
+temp = require 'temp'
+fstream = require 'fstream'
 Project = require '../src/project'
 {_, fs} = require 'atom'
 path = require 'path'
@@ -232,33 +234,37 @@ describe "Project", ->
 
   describe ".scan(options, callback)", ->
     describe "when called with a regex", ->
-      it "calls the callback with all regex matches in all files in the project", ->
-        matches = []
+      it "calls the callback with all regex results in all files in the project", ->
+        results = []
         waitsForPromise ->
-          project.scan /(a)+/, (match) -> matches.push(match)
+          project.scan /(a)+/, (result) ->
+            results.push(result)
 
         runs ->
-          expect(matches[0]).toEqual
-            path: project.resolve('a')
-            match: 'aaa'
+          expect(results).toHaveLength(3)
+          expect(results[0].filePath).toBe project.resolve('a')
+          expect(results[0].matches).toHaveLength(3)
+          expect(results[0].matches[0]).toEqual
+            matchText: 'aaa'
+            lineText: 'aaa bbb'
+            lineTextOffset: 0
             range: [[0, 0], [0, 3]]
 
-          expect(matches[1]).toEqual
-            path: project.resolve('a')
-            match: 'aa'
-            range: [[1, 3], [1, 5]]
-
       it "works with with escaped literals (like $ and ^)", ->
-        matches = []
+        results = []
         waitsForPromise ->
-          project.scan /\$\w+/, (match) -> matches.push(match)
+          project.scan /\$\w+/, (result) -> results.push(result)
 
         runs ->
-          expect(matches.length).toBe 1
+          expect(results.length).toBe 1
 
+          {filePath, matches} = results[0]
+          expect(filePath).toBe project.resolve('a')
+          expect(matches).toHaveLength 1
           expect(matches[0]).toEqual
-            path: project.resolve('a')
-            match: '$bill'
+            matchText: '$bill'
+            lineText: 'dollar$bill'
+            lineTextOffset: 0
             range: [[2, 6], [2, 11]]
 
       it "works on evil filenames", ->
@@ -267,12 +273,12 @@ describe "Project", ->
         matches = []
         waitsForPromise ->
           project.scan /evil/, (result) ->
-            paths.push(result.path)
-            matches.push(result.match)
+            paths.push(result.filePath)
+            matches = matches.concat(result.matches)
 
         runs ->
           expect(paths.length).toBe 5
-          matches.forEach (match) -> expect(match).toEqual 'evil'
+          matches.forEach (match) -> expect(match.matchText).toEqual 'evil'
           expect(paths[0]).toMatch /a_file_with_utf8.txt$/
           expect(paths[1]).toMatch /file with spaces.txt$/
           expect(paths[2]).toMatch /goddam\nnewlines$/m
@@ -280,57 +286,63 @@ describe "Project", ->
           expect(path.basename(paths[4])).toBe "utfa\u0306.md"
 
       it "ignores case if the regex includes the `i` flag", ->
-        matches = []
+        results = []
         waitsForPromise ->
-          project.scan /DOLLAR/i, (match) -> matches.push(match)
+          project.scan /DOLLAR/i, (result) -> results.push(result)
 
         runs ->
-          expect(matches).toHaveLength 1
-
-      it "handles breaks in the search subprocess's output following the filename", ->
-        spyOn(BufferedProcess.prototype, 'bufferStream')
-
-        iterator = jasmine.createSpy('iterator')
-        project.scan /a+/, iterator
-
-        stdout = BufferedProcess.prototype.bufferStream.argsForCall[0][1]
-        stdout ":#{path.join(__dirname, 'fixtures', 'dir', 'a')}\n"
-        stdout "1;0 3:aaa bbb\n2;3 2:cc aa cc\n"
-
-        expect(iterator.argsForCall[0][0]).toEqual
-          path: project.resolve('a')
-          match: 'aaa'
-          range: [[0, 0], [0, 3]]
-
-        expect(iterator.argsForCall[1][0]).toEqual
-          path: project.resolve('a')
-          match: 'aa'
-          range: [[1, 3], [1, 5]]
+          expect(results).toHaveLength 1
 
       describe "when the core.excludeVcsIgnoredPaths config is truthy", ->
         [projectPath, ignoredPath] = []
 
         beforeEach ->
-          projectPath = path.join(__dirname, 'fixtures', 'git', 'working-dir')
-          ignoredPath = path.join(projectPath, 'ignored.txt')
-          fs.writeSync(ignoredPath, 'this match should not be included')
+          sourceProjectPath = path.join(__dirname, 'fixtures', 'git', 'working-dir')
+          projectPath = path.join(temp.mkdirSync("atom"))
+
+          writerStream = fstream.Writer(projectPath)
+          fstream.Reader(sourceProjectPath).pipe(writerStream)
+
+          waitsFor (done) ->
+            writerStream.on 'close', done
+            writerStream.on 'error', done
+
+          runs ->
+            fs.rename(path.join(projectPath, 'git.git'), path.join(projectPath, '.git'))
+            ignoredPath = path.join(projectPath, 'ignored.txt')
+            fs.writeSync(ignoredPath, 'this match should not be included')
 
         afterEach ->
-          fs.remove(ignoredPath) if fs.exists(ignoredPath)
+          fs.remove(projectPath) if fs.exists(projectPath)
 
         it "excludes ignored files", ->
           project.setPath(projectPath)
           config.set('core.excludeVcsIgnoredPaths', true)
-          paths = []
-          matches = []
+          resultHandler = jasmine.createSpy("result found")
           waitsForPromise ->
-            project.scan /match/, (result) ->
-              paths.push(result.path)
-              matches.push(result.match)
+            project.scan /match/, (results) ->
+              resultHandler()
 
           runs ->
-            expect(paths.length).toBe 0
-            expect(matches.length).toBe 0
+            expect(resultHandler).not.toHaveBeenCalled()
+
+      it "includes only files when a directory filter is specified", ->
+        projectPath = path.join(path.join(__dirname, 'fixtures', 'dir'))
+        project.setPath(projectPath)
+
+        filePath = path.join(projectPath, 'a-dir', 'oh-git')
+
+        paths = []
+        matches = []
+        waitsForPromise ->
+          project.scan /aaa/, paths: ['a-dir/'], (result) ->
+            paths.push(result.filePath)
+            matches = matches.concat(result.matches)
+
+        runs ->
+          expect(paths.length).toBe 1
+          expect(paths[0]).toBe filePath
+          expect(matches.length).toBe 1
 
       it "includes files and folders that begin with a '.'", ->
         projectPath = '/tmp/atom-tests/folder-with-dot-file'
@@ -341,8 +353,8 @@ describe "Project", ->
         matches = []
         waitsForPromise ->
           project.scan /match this/, (result) ->
-            paths.push(result.path)
-            matches.push(result.match)
+            paths.push(result.filePath)
+            matches = matches.concat(result.matches)
 
         runs ->
           expect(paths.length).toBe 1
@@ -350,17 +362,16 @@ describe "Project", ->
           expect(matches.length).toBe 1
 
       it "excludes values in core.ignoredNames", ->
-        projectPath = '/tmp/atom-tests/folder-with-dot-git/.git'
-        filePath = path.join(projectPath, 'test.txt')
-        fs.writeSync(filePath, 'match this')
-        project.setPath(projectPath)
-        paths = []
-        matches = []
+        projectPath = path.join(__dirname, 'fixtures', 'git', 'working-dir')
+        ignoredNames = config.get("core.ignoredNames")
+        ignoredNames.push("a")
+        config.set("core.ignoredNames", ignoredNames)
+
+        resultHandler = jasmine.createSpy("result found")
         waitsForPromise ->
-          project.scan /match/, (result) ->
-            paths.push(result.path)
-            matches.push(result.match)
+          project.scan /dollar/, (results) ->
+            console.log results
+            resultHandler()
 
         runs ->
-          expect(paths.length).toBe 0
-          expect(matches.length).toBe 0
+          expect(resultHandler).not.toHaveBeenCalled()

spec/text-buffer-spec.coffee

@@ -602,6 +602,17 @@ describe 'TextBuffer', ->
       it "clips the range to the end of the buffer", ->
         expect(buffer.getTextInRange([[12], [13, Infinity]])).toBe buffer.lineForRow(12)
 
+  describe ".scan(regex, fn)", ->
+    it "retunrns lineText and lineTextOffset", ->
+      matches = []
+      buffer.scan /current/, (match) ->
+        matches.push(match)
+      expect(matches.length).toBe 1
+
+      expect(matches[0].matchText).toEqual 'current'
+      expect(matches[0].lineText).toEqual '    var pivot = items.shift(), current, left = [], right = [];'
+      expect(matches[0].lineTextOffset).toBe 0
+
   describe ".scanInRange(range, regex, fn)", ->
     describe "when given a regex with a ignore case flag", ->
       it "does a case-insensitive search", ->

src/edit-session.coffee

@@ -117,6 +117,7 @@ class EditSession
       project.setPath(path.dirname(@getPath())) unless project.getPath()?
       @trigger "title-changed"
       @trigger "path-changed"
+    @subscribe @buffer, "contents-modified", => @trigger "contents-modified"
     @subscribe @buffer, "contents-conflicted", => @trigger "contents-conflicted"
     @subscribe @buffer, "modified-status-changed", => @trigger "modified-status-changed"
     @preserveCursorPositionOnBufferReload()
@@ -376,6 +377,9 @@ class EditSession
   # {Delegates to: TextBuffer.lineLengthForRow}
   lineLengthForBufferRow: (row) -> @buffer.lineLengthForRow(row)
 
+  # {Delegates to: TextBuffer.scan}
+  scan: (args...) -> @buffer.scan(args...)
+
   # {Delegates to: TextBuffer.scanInRange}
   scanInBufferRange: (args...) -> @buffer.scanInRange(args...)
 

src/project.coffee

@@ -10,7 +10,7 @@ TextBuffer = require './text-buffer'
 EditSession = require './edit-session'
 EventEmitter = require './event-emitter'
 Directory = require './directory'
-BufferedNodeProcess = require './buffered-node-process'
+Task = require './task'
 Git = require './git'
 
 # Public: Represents a project that's opened in Atom.
@@ -278,65 +278,30 @@ class Project
   #
   # * regex:
   #   A RegExp to search with
+  # * options:
+  #   - paths: an {Array} of glob patterns to search within
   # * iterator:
   #   A Function callback on each file found
-  scan: (regex, iterator) ->
-    bufferedData = ""
-    state = 'readingPath'
-    filePath = null
-
-    readPath = (line) ->
-      if /^[0-9,; ]+:/.test(line)
-        state = 'readingLines'
-      else if /^:/.test line
-        filePath = line.substr(1)
-      else
-        filePath += ('\n' + line)
-
-    readLine = (line) ->
-      if line.length == 0
-        state = 'readingPath'
-        filePath = null
-      else
-        colonIndex = line.indexOf(':')
-        matchInfo = line.substring(0, colonIndex)
-        lineText = line.substring(colonIndex + 1)
-        readMatches(matchInfo, lineText)
-
-    readMatches = (matchInfo, lineText) ->
-      [lineNumber, matchPositionsText] = matchInfo.match(/(\d+);(.+)/)[1..]
-      row = parseInt(lineNumber) - 1
-      matchPositions = matchPositionsText.split(',').map (positionText) -> positionText.split(' ').map (pos) -> parseInt(pos)
-
-      for [column, length] in matchPositions
-        range = new Range([row, column], [row, column + length])
-        match = lineText.substr(column, length)
-        iterator({path: filePath, range, match})
+  scan: (regex, options={}, iterator) ->
+    if _.isFunction(options)
+      iterator = options
+      options = {}
 
     deferred = $.Deferred()
-    errors = []
-    stderr = (data) ->
-      errors.push(data)
-    stdout = (data) ->
-      lines = data.split('\n')
-      lines.pop() # the last segment is a spurious '' because data always ends in \n due to bufferLines: true
-      for line in lines
-        readPath(line) if state is 'readingPath'
-        readLine(line) if state is 'readingLines'
-    exit = (code) ->
-      if code is 0
-        deferred.resolve()
-      else
-        console.error("Project scan failed: #{code}", errors.join('\n'))
-        deferred.reject({command, code})
 
-    command = require.resolve('.bin/nak')
-    args = ['--hidden', '--ackmate', regex.source, @getPath()]
-    ignoredNames = config.get('core.ignoredNames') ? []
-    args.unshift('--ignore', ignoredNames.join(',')) if ignoredNames.length > 0
-    args.unshift('--ignoreCase') if regex.ignoreCase
-    args.unshift('--addVCSIgnores') if config.get('core.excludeVcsIgnoredPaths')
-    new BufferedNodeProcess({command, args, stdout, stderr, exit})
+    searchOptions =
+      ignoreCase: regex.ignoreCase
+      inclusions: options.paths
+      includeHidden: true
+      excludeVcsIgnores: config.get('core.excludeVcsIgnoredPaths')
+      exclusions: config.get('core.ignoredNames')
+
+    task = Task.once require.resolve('./scan-handler'), @getPath(), regex.source, searchOptions, ->
+      deferred.resolve()
+
+    task.on 'scan:result-found', (result) =>
+      iterator(result)
+
     deferred
 
   # Private:

src/scan-handler.coffee

@@ -0,0 +1,15 @@
+{PathSearcher, PathScanner, search} = require 'scandal'
+
+module.exports = (rootPath, regexSource, options) ->
+  callback = @async()
+
+  searcher = new PathSearcher()
+  scanner = new PathScanner(rootPath, options)
+
+  searcher.on 'results-found', (result) ->
+    emit('scan:result-found', result)
+
+  flags = "g"
+  flags += "i" if options.ignoreCase
+  regex = new RegExp(regexSource, flags)
+  search regex, scanner, searcher, callback

src/text-buffer.coffee

@@ -6,6 +6,7 @@ File = require './file'
 EventEmitter = require './event-emitter'
 Subscriber = require './subscriber'
 guid = require 'guid'
+{P} = require 'scandal'
 
 # Private: Represents the contents of a file.
 #
@@ -501,7 +502,10 @@ class TextBuffer
   # regex - A {RegExp} representing the text to find
   # iterator - A {Function} that's called on each match
   scan: (regex, iterator) ->
-    @scanInRange(regex, @getRange(), iterator)
+    @scanInRange regex, @getRange(), (result) =>
+      result.lineText = @lineForRow(result.range.start.row)
+      result.lineTextOffset = 0
+      iterator(result)
 
   # Scans for text in a given range, calling a function on each match.
   #
@@ -538,7 +542,8 @@ class TextBuffer
       range = new Range(startPosition, endPosition)
       keepLooping = true
       replacementText = null
-      iterator({match, range, stop, replace })
+      matchText = match[0]
+      iterator({ match, matchText, range, stop, replace })
 
       if replacementText?
         @change(range, replacementText)

src/underscore-extensions.coffee

@@ -28,6 +28,9 @@ _.mixin
   escapeRegExp: (string) ->
     string.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&')
 
+  escapeAttribute: (string) ->
+    string.replace(/"/g, '"').replace(/\n/g, '')
+
   humanizeEventName: (eventName, eventDoc) ->
     [namespace, event]  = eventName.split(':')
     return _.undasherize(namespace) unless event?