[CS2] Add #! support for executable scripts on Linux. (#3946)

* Add #! support for executable scripts on Linux.

Pass arguments to executable script unchanged if using "#!/usr/bin/env
coffee". (Previously, "./test.coffee -abck" would be turned into "-a -b -c -k",
for example.)

Fixes #1752.

* refactor option parsing

clean up parsing code and in the process fix oustanding bug where coffeescript
modified arguments meant for an executable script

* address comments

* intermediate save

* add note saying where OptionParser is used in coffee command

* add some more work

* fix flatten functions

* refactor tests

* make argument processing less confusing

* add basic test

* remove unused file

* compilation now hangs

* remove unnecessary changes

* add tests!!!

* add/fix some tests

* clarify a test

* fix helpers

* fix opt parsing

* fix infinite loop

* make rule building easier to read

* add tests for flag overlap

* revamp argument parsing again and add more thorough testing

* add tests, comment, clean unused method

* address review comments

* add test for direct invocation of shebang scripts

* move shebang parsing test to separate file and check for browser

* remove TODO

* example backwards compatible warnings

* add correct tests for warning 1

* add tests for warnings

* commit output js libs and update docs

* respond to review comments

also add tests for help text

* respond to review comments

* fix example output

* Rewrite argument parsing documentation to be more concise; add it to sidebar and body; add new output

* Don’t mention deprecated syntax; clean up variable names

test/argument-parsing.coffee

@@ -1,75 +0,0 @@
-return unless require?
-
-{buildCSOptionParser} = require '../lib/coffeescript/command'
-
-optionParser = buildCSOptionParser()
-
-sameOptions = (opts1, opts2, msg) ->
-  ownKeys = Object.keys(opts1).sort()
-  otherKeys = Object.keys(opts2).sort()
-  arrayEq ownKeys, otherKeys, msg
-  for k in ownKeys
-    arrayEq opts1[k], opts2[k], msg
-  yes
-
-test "combined options are still split after initial file name", ->
-  argv = ['some-file.coffee', '-bc']
-  parsed = optionParser.parse argv
-  expected = arguments: ['some-file.coffee', '-b', '-c']
-  sameOptions parsed, expected
-
-  argv = ['some-file.litcoffee', '-bc']
-  parsed = optionParser.parse argv
-  expected = arguments: ['some-file.litcoffee', '-b', '-c']
-  sameOptions parsed, expected
-
-  argv = ['-c', 'some-file.coffee', '-bc']
-  parsed = optionParser.parse argv
-  expected =
-    compile: yes
-    arguments: ['some-file.coffee', '-b', '-c']
-  sameOptions parsed, expected
-
-  argv = ['-bc', 'some-file.coffee', '-bc']
-  parsed = optionParser.parse argv
-  expected =
-    bare: yes
-    compile: yes
-    arguments: ['some-file.coffee', '-b', '-c']
-  sameOptions parsed, expected
-
-test "combined options are not split after a '--'", ->
-  argv = ['--', '-bc']
-  parsed = optionParser.parse argv
-  expected = arguments: ['-bc']
-  sameOptions parsed, expected
-
-  argv = ['-bc', '--', '-bc']
-  parsed = optionParser.parse argv
-  expected =
-    bare: yes
-    compile: yes
-    arguments: ['-bc']
-  sameOptions parsed, expected
-
-test "options are not split after any '--'", ->
-  argv = ['--', '--', '-bc']
-  parsed = optionParser.parse argv
-  expected = arguments: ['--', '-bc']
-  sameOptions parsed, expected
-
-  argv = ['--', 'some-file.coffee', '--', 'arg']
-  parsed = optionParser.parse argv
-  expected = arguments: ['some-file.coffee', '--', 'arg']
-  sameOptions parsed, expected
-
-  argv = ['--', 'arg', 'some-file.coffee', '--', '-bc']
-  parsed = optionParser.parse argv
-  expected = arguments: ['arg', 'some-file.coffee', '--', '-bc']
-  sameOptions parsed, expected
-
-test "later '--' are removed", ->
-  argv = ['some-file.coffee', '--', '-bc']
-  parsed = optionParser.parse argv
-  expected = arguments: ['some-file.coffee', '-bc']
-  sameOptions parsed, expected

test/argument_parsing.coffee

@@ -0,0 +1,142 @@
+return unless require?
+{buildCSOptionParser} = require '../lib/coffeescript/command'
+
+optionParser = buildCSOptionParser()
+
+sameOptions = (opts1, opts2, msg) ->
+  ownKeys = Object.keys(opts1).sort()
+  otherKeys = Object.keys(opts2).sort()
+  arrayEq ownKeys, otherKeys, msg
+  for k in ownKeys
+    arrayEq opts1[k], opts2[k], msg
+  yes
+
+test "combined options are not split after initial file name", ->
+  argv = ['some-file.coffee', '-bc']
+  parsed = optionParser.parse argv
+  expected = arguments: ['some-file.coffee', '-bc']
+  sameOptions parsed, expected
+
+  argv = ['some-file.litcoffee', '-bc']
+  parsed = optionParser.parse argv
+  expected = arguments: ['some-file.litcoffee', '-bc']
+  sameOptions parsed, expected
+
+  argv = ['-c', 'some-file.coffee', '-bc']
+  parsed = optionParser.parse argv
+  expected =
+    compile: yes
+    arguments: ['some-file.coffee', '-bc']
+  sameOptions parsed, expected
+
+  argv = ['-bc', 'some-file.coffee', '-bc']
+  parsed = optionParser.parse argv
+  expected =
+    bare: yes
+    compile: yes
+    arguments: ['some-file.coffee', '-bc']
+  sameOptions parsed, expected
+
+test "combined options are not split after a '--', which is discarded", ->
+  argv = ['--', '-bc']
+  parsed = optionParser.parse argv
+  expected =
+    doubleDashed: yes
+    arguments: ['-bc']
+  sameOptions parsed, expected
+
+  argv = ['-bc', '--', '-bc']
+  parsed = optionParser.parse argv
+  expected =
+    bare: yes
+    compile: yes
+    doubleDashed: yes
+    arguments: ['-bc']
+  sameOptions parsed, expected
+
+test "options are not split after any '--'", ->
+  argv = ['--', '--', '-bc']
+  parsed = optionParser.parse argv
+  expected =
+    doubleDashed: yes
+    arguments: ['--', '-bc']
+  sameOptions parsed, expected
+
+  argv = ['--', 'some-file.coffee', '--', 'arg']
+  parsed = optionParser.parse argv
+  expected =
+    doubleDashed: yes
+    arguments: ['some-file.coffee', '--', 'arg']
+  sameOptions parsed, expected
+
+  argv = ['--', 'arg', 'some-file.coffee', '--', '-bc']
+  parsed = optionParser.parse argv
+  expected =
+    doubleDashed: yes
+    arguments: ['arg', 'some-file.coffee', '--', '-bc']
+  sameOptions parsed, expected
+
+test "any non-option argument stops argument parsing", ->
+  argv = ['arg', '-bc']
+  parsed = optionParser.parse argv
+  expected = arguments: ['arg', '-bc']
+  sameOptions parsed, expected
+
+test "later '--' are not removed", ->
+  argv = ['some-file.coffee', '--', '-bc']
+  parsed = optionParser.parse argv
+  expected = arguments: ['some-file.coffee', '--', '-bc']
+  sameOptions parsed, expected
+
+test "throw on invalid options", ->
+  argv = ['-k']
+  throws -> optionParser.parse argv
+
+  argv = ['-ck']
+  throws (-> optionParser.parse argv), /multi-flag/
+
+  argv = ['-kc']
+  throws (-> optionParser.parse argv), /multi-flag/
+
+  argv = ['-oc']
+  throws (-> optionParser.parse argv), /needs an argument/
+
+  argv = ['-o']
+  throws (-> optionParser.parse argv), /value required/
+
+  argv = ['-co']
+  throws (-> optionParser.parse argv), /value required/
+
+  # Check if all flags in a multi-flag are recognized before checking if flags
+  # before the last need arguments.
+  argv = ['-ok']
+  throws (-> optionParser.parse argv), /unrecognized option/
+
+test "has expected help text", ->
+  ok optionParser.help() is '''
+
+Usage: coffee [options] path/to/script.coffee [args]
+
+If called without options, `coffee` will run your script.
+
+  -b, --bare         compile without a top-level function wrapper
+  -c, --compile      compile to JavaScript and save as .js files
+  -e, --eval         pass a string from the command line as input
+  -h, --help         display this help message
+  -i, --interactive  run an interactive CoffeeScript REPL
+  -j, --join         concatenate the source CoffeeScript before compiling
+  -m, --map          generate source map and save as .js.map files
+  -M, --inline-map   generate source map and include it directly in output
+  -n, --nodes        print out the parse tree that the parser produces
+      --nodejs       pass options directly to the "node" binary
+      --no-header    suppress the "Generated by" header
+  -o, --output       set the output directory for compiled JavaScript
+  -p, --print        print out the compiled JavaScript
+  -r, --require      require the given module before eval or REPL
+  -s, --stdio        listen for and compile scripts over stdio
+  -l, --literate     treat stdio as literate style coffeescript
+  -t, --tokens       print out the tokens that the lexer/rewriter produce
+  -v, --version      display the version number
+  -w, --watch        watch scripts for changes and rerun commands
+
+  '''

test/importing/shebang.coffee

@@ -0,0 +1,3 @@
+#!/usr/bin/env coffee
+
+process.stdout.write JSON.stringify(process.argv)

test/importing/shebang_extra_args.coffee

@@ -0,0 +1,3 @@
+#!/usr/bin/env coffee --
+
+process.stdout.write JSON.stringify(process.argv)

test/importing/shebang_initial_space.coffee

@@ -0,0 +1,3 @@
+#!  /usr/bin/env coffee
+
+process.stdout.write JSON.stringify(process.argv)

test/importing/shebang_initial_space_extra_args.coffee

@@ -0,0 +1,3 @@
+#! /usr/bin/env coffee extra
+
+process.stdout.write JSON.stringify(process.argv)

test/invocation_argument_parsing.coffee

@@ -0,0 +1,108 @@
+return unless require?
+
+path = require 'path'
+{spawnSync, execFileSync} = require 'child_process'
+
+# Get directory containing the compiled `coffee` executable and prepend it to
+# the path so `#!/usr/bin/env coffee` resolves to our locally built file.
+coffeeBinDir = path.dirname require.resolve('../bin/coffee')
+patchedPath = "#{coffeeBinDir}:#{process.env.PATH}"
+patchedEnv = Object.assign {}, process.env, {PATH: patchedPath}
+
+shebangScript = require.resolve './importing/shebang.coffee'
+initialSpaceScript = require.resolve './importing/shebang_initial_space.coffee'
+extraArgsScript = require.resolve './importing/shebang_extra_args.coffee'
+initialSpaceExtraArgsScript = require.resolve './importing/shebang_initial_space_extra_args.coffee'
+
+test "parse arguments for shebang scripts correctly (on unix platforms)", ->
+  return if isWindows()
+
+  stdout = execFileSync shebangScript, ['-abck'], {env: patchedEnv}
+  expectedArgs = ['coffee', shebangScript, '-abck']
+  realArgs = JSON.parse stdout
+  arrayEq expectedArgs, realArgs
+
+  stdout = execFileSync initialSpaceScript, ['-abck'], {env: patchedEnv}
+  expectedArgs = ['coffee', initialSpaceScript, '-abck']
+  realArgs = JSON.parse stdout
+  arrayEq expectedArgs, realArgs
+
+test "warn and remove -- if it is the second positional argument", ->
+  result = spawnSync 'coffee', [shebangScript, '--'], {env: patchedEnv}
+  stderr = result.stderr.toString()
+  arrayEq JSON.parse(result.stdout), ['coffee', shebangScript]
+  ok stderr.match /^coffee was invoked with '--'/m
+  posArgs = stderr.match(/^The positional arguments were: (.*)$/m)[1]
+  arrayEq JSON.parse(posArgs), [shebangScript, '--']
+  ok result.status is 0
+
+  result = spawnSync 'coffee', ['-b', shebangScript, '--'], {env: patchedEnv}
+  stderr = result.stderr.toString()
+  arrayEq JSON.parse(result.stdout), ['coffee', shebangScript]
+  ok stderr.match /^coffee was invoked with '--'/m
+  posArgs = stderr.match(/^The positional arguments were: (.*)$/m)[1]
+  arrayEq JSON.parse(posArgs), [shebangScript, '--']
+  ok result.status is 0
+
+  result = spawnSync(
+    'coffee', ['-b', shebangScript, '--', 'ANOTHER ONE'], {env: patchedEnv})
+  stderr = result.stderr.toString()
+  arrayEq JSON.parse(result.stdout), ['coffee', shebangScript, 'ANOTHER ONE']
+  ok stderr.match /^coffee was invoked with '--'/m
+  posArgs = stderr.match(/^The positional arguments were: (.*)$/m)[1]
+  arrayEq JSON.parse(posArgs), [shebangScript, '--', 'ANOTHER ONE']
+  ok result.status is 0
+
+  result = spawnSync(
+    'coffee', ['--', initialSpaceScript, 'arg'], {env: patchedEnv})
+  expectedArgs = ['coffee', initialSpaceScript, 'arg']
+  realArgs = JSON.parse result.stdout
+  arrayEq expectedArgs, realArgs
+  ok result.stderr.toString() is ''
+  ok result.status is 0
+
+test "warn about non-portable shebang lines", ->
+  result = spawnSync 'coffee', [extraArgsScript, 'arg'], {env: patchedEnv}
+  stderr = result.stderr.toString()
+  arrayEq JSON.parse(result.stdout), ['coffee', extraArgsScript, 'arg']
+  ok stderr.match /^The script to be run begins with a shebang line with more than one/m
+  [_, firstLine, file] = stderr.match(/^The shebang line was: '([^']+)' in file '([^']+)'/m)
+  ok (firstLine is '#!/usr/bin/env coffee --')
+  ok (file is extraArgsScript)
+  args = stderr.match(/^The arguments were: (.*)$/m)[1]
+  arrayEq JSON.parse(args), ['coffee', '--']
+  ok result.status is 0
+
+  result = spawnSync 'coffee', [initialSpaceScript, 'arg'], {env: patchedEnv}
+  stderr = result.stderr.toString()
+  ok stderr is ''
+  arrayEq JSON.parse(result.stdout), ['coffee', initialSpaceScript, 'arg']
+  ok result.status is 0
+
+  result = spawnSync(
+    'coffee', [initialSpaceExtraArgsScript, 'arg'], {env: patchedEnv})
+  stderr = result.stderr.toString()
+  arrayEq JSON.parse(result.stdout), ['coffee', initialSpaceExtraArgsScript, 'arg']
+  ok stderr.match /^The script to be run begins with a shebang line with more than one/m
+  [_, firstLine, file] = stderr.match(/^The shebang line was: '([^']+)' in file '([^']+)'/m)
+  ok (firstLine is '#! /usr/bin/env coffee extra')
+  ok (file is initialSpaceExtraArgsScript)
+  args = stderr.match(/^The arguments were: (.*)$/m)[1]
+  arrayEq JSON.parse(args), ['coffee', 'extra']
+  ok result.status is 0
+
+test "both warnings will be shown at once", ->
+  result = spawnSync(
+    'coffee', [initialSpaceExtraArgsScript, '--', 'arg'], {env: patchedEnv})
+  stderr = result.stderr.toString()
+  arrayEq JSON.parse(result.stdout), ['coffee', initialSpaceExtraArgsScript, 'arg']
+  ok stderr.match /^The script to be run begins with a shebang line with more than one/m
+  [_, firstLine, file] = stderr.match(/^The shebang line was: '([^']+)' in file '([^']+)'/m)
+  ok (firstLine is '#! /usr/bin/env coffee extra')
+  ok (file is initialSpaceExtraArgsScript)
+  args = stderr.match(/^The arguments were: (.*)$/m)[1]
+  arrayEq JSON.parse(args), ['coffee', 'extra']
+  ok stderr.match /^coffee was invoked with '--'/m
+  posArgs = stderr.match(/^The positional arguments were: (.*)$/m)[1]
+  arrayEq JSON.parse(posArgs), [initialSpaceExtraArgsScript, '--', 'arg']
+  ok result.status is 0

test/option_parser.coffee

@@ -1,18 +1,22 @@
 # Option Parser
 # -------------
 
-# TODO: refactor option parser tests
-
 # Ensure that the OptionParser handles arguments correctly.
 return unless require?
 {OptionParser} = require './../lib/coffeescript/optparse'
 
-opt = new OptionParser [
+flags = [
   ['-r', '--required [DIR]',  'desc required']
   ['-o', '--optional',        'desc optional']
   ['-l', '--list [FILES*]',   'desc list']
 ]
 
+banner = '''
+  banner text
+'''
+
+opt = new OptionParser flags, banner
+
 test "basic arguments", ->
   args = ['one', 'two', 'three', '-r', 'dir']
   result = opt.parse args
@@ -41,3 +45,50 @@ test "-- and interesting combinations", ->
   eq undefined, result.optional
   eq undefined, result.required
   arrayEq args[1..], result.arguments
+
+test "throw if multiple flags try to use the same short or long name", ->
+  throws -> new OptionParser [
+    ['-r', '--required [DIR]', 'required']
+    ['-r', '--long',           'switch']
+  ]
+
+  throws -> new OptionParser [
+    ['-a', '--append [STR]', 'append']
+    ['-b', '--append',       'append with -b short opt']
+  ]
+
+  throws -> new OptionParser [
+    ['--just-long', 'desc']
+    ['--just-long', 'another desc']
+  ]
+
+  throws -> new OptionParser [
+    ['-j', '--just-long', 'desc']
+    ['--just-long', 'another desc']
+  ]
+
+  throws -> new OptionParser [
+    ['--just-long',       'desc']
+    ['-j', '--just-long', 'another desc']
+  ]
+
+test "outputs expected help text", ->
+  expectedBanner = '''
+
+banner text
+
+  -r, --required     desc required
+  -o, --optional     desc optional
+  -l, --list         desc list
+
+  '''
+  ok opt.help() is expectedBanner
+
+  expected = [
+    ''
+    '  -r, --required     desc required'
+    '  -o, --optional     desc optional'
+    '  -l, --list         desc list'
+    ''
+  ].join('\n')
+  ok new OptionParser(flags).help() is expected

test/support/helpers.coffee

@@ -30,3 +30,5 @@ exports.eqJS = (input, expectedOutput, msg) ->
   #{reset}#{expectedOutput}#{red}
     but instead it was:
   #{reset}#{actualOutput}#{red}"""
+
+exports.isWindows = -> process.platform is 'win32'