diff --git a/src/browser/atom-application.coffee b/src/browser/atom-application.coffee
index b8e3e7bc2..ab7a45db9 100644
--- a/src/browser/atom-application.coffee
+++ b/src/browser/atom-application.coffee
@@ -91,7 +91,7 @@ class AtomApplication
 
   openWithOptions: ({pathsToOpen, executedFrom, urlsToOpen, test, pidToKillWhenClosed, devMode, safeMode, newWindow, logFile, profileStartup, timeout}) ->
     if test
-      @runTests({headless: true, @resourcePath, executedFrom, pathsToOpen, logFile, timeout})
+      @runTests({headless: true, devMode, @resourcePath, executedFrom, pathsToOpen, logFile, timeout})
     else if pathsToOpen.length > 0
       @openPaths({pathsToOpen, executedFrom, pidToKillWhenClosed, newWindow, devMode, safeMode, profileStartup})
     else if urlsToOpen.length > 0
@@ -494,7 +494,7 @@ class AtomApplication
   #   :specPath - The directory to load specs from.
   #   :safeMode - A Boolean that, if true, won't run specs from ~/.atom/packages
   #               and ~/.atom/dev/packages, defaults to false.
-  runTests: ({headless, resourcePath, executedFrom, pathsToOpen, logFile, safeMode, timeout}) ->
+  runTests: ({headless, devMode, resourcePath, executedFrom, pathsToOpen, logFile, safeMode, timeout}) ->
     if resourcePath isnt @resourcePath and not fs.existsSync(resourcePath)
       resourcePath = @resourcePath
 
@@ -522,7 +522,6 @@ class AtomApplication
     legacyTestRunnerPath = @resolveLegacyTestRunnerPath()
     testRunnerPath = @resolveTestRunnerPath(testPaths[0])
     isSpec = true
-    devMode = true
     safeMode ?= false
     new AtomWindow({windowInitializationScript, resourcePath, headless, isSpec, devMode, testRunnerPath, legacyTestRunnerPath, testPaths, logFile, safeMode})
 
diff --git a/src/browser/main.coffee b/src/browser/main.coffee
index 6a141cf1a..990002671 100644
--- a/src/browser/main.coffee
+++ b/src/browser/main.coffee
@@ -133,6 +133,8 @@ parseCommandLine = ->
   if args['resource-path']
     devMode = true
     resourcePath = args['resource-path']
+
+  devMode = true if test
   resourcePath ?= devResourcePath if devMode
 
   unless fs.statSyncNoException(resourcePath)
diff --git a/src/initialize-test-window.coffee b/src/initialize-test-window.coffee
index ad15429ff..27ef46acc 100644
--- a/src/initialize-test-window.coffee
+++ b/src/initialize-test-window.coffee
@@ -13,9 +13,21 @@ try
   AtomEnvironment = require '../src/atom-environment'
   ApplicationDelegate = require '../src/application-delegate'
 
-  # Show window synchronously so a focusout doesn't fire on input elements
-  # that are focused in the very first spec run.
-  remote.getCurrentWindow().show() unless getWindowLoadSettings().headless
+  {testRunnerPath, legacyTestRunnerPath, headless, logFile, testPaths} = getWindowLoadSettings()
+
+  if headless
+    # Override logging in headless mode so it goes to the console, regardless
+    # of the --enable-logging flag to Electron.
+    console.log = (args...) ->
+      ipc.send 'write-to-stdout', args.join(' ') + '\n'
+    console.warn = (args...) ->
+      ipc.send 'write-to-stderr', args.join(' ') + '\n'
+    console.error = (args...) ->
+      ipc.send 'write-to-stderr', args.join(' ') + '\n'
+  else
+    # Show window synchronously so a focusout doesn't fire on input elements
+    # that are focused in the very first spec run.
+    remote.getCurrentWindow().show()
 
   handleKeydown = (event) ->
     # Reload: cmd-r / ctrl-r
@@ -39,15 +51,13 @@ try
 
   document.title = "Spec Suite"
 
-  legacyTestRunner = require(getWindowLoadSettings().legacyTestRunnerPath)
-  testRunner = require(getWindowLoadSettings().testRunnerPath)
+  testRunner = require(testRunnerPath)
+  legacyTestRunner = require(legacyTestRunnerPath)
+  buildAtomEnvironment = (params) -> new AtomEnvironment(params)
+  buildDefaultApplicationDelegate = (params) -> new ApplicationDelegate()
+
   promise = testRunner({
-    logFile: getWindowLoadSettings().logFile
-    headless: getWindowLoadSettings().headless
-    testPaths: getWindowLoadSettings().testPaths
-    buildAtomEnvironment: (params) -> new AtomEnvironment(params)
-    buildDefaultApplicationDelegate: (params) -> new ApplicationDelegate()
-    legacyTestRunner: legacyTestRunner
+    logFile, headless, testPaths, buildAtomEnvironment, buildDefaultApplicationDelegate, legacyTestRunner
   })
 
   promise.then(exitWithStatusCode) if getWindowLoadSettings().headless