Support fetching dynamic modules via HTTP POST requests.

Ever since Meteor 1.5 first shipped, dynamic modules have been fetched
over a WebSocket, which is appealing because sockets have very little
latency and metadata overhead per round-trip.

However, using a WebSocket requires first establishing a socket connection
to the server, which takes time and may require a WebSocket polyfill.

An even more subtle problem is that we cannot use dynamic imports in any
of the code that implements the ddp-client package, as long as the
dynamic-import package depends on ddp-client.

By switching from WebSockets to HTTP POST requests, this commit radically
reduces the dependencies of the dynamic-import package, while preserving
(or even exceeding) the benefits of socket-based dynamic module fetching:

1. The client makes a single HTTP POST request for the exact set of
   dynamic modules that it needs, which is much cheaper/faster than making
   several/many HTTP requests in parallel, with or without HTTP/2.

2. Because the client uses a permanent cache (IndexedDB) to avoid
   requesting any modules it has ever previously received, the lack of
   HTTP caching for POST requests is not a problem.

3. Because the HTTP response contains all the requested dynamic modules in
   a single JSON payload, gzip compression works across modules, which
   produces a smaller total response size than if each individual module
   was compressed by itself.

4. Although HTTP requests have more latency than WebSocket messages, the
   ability to make HTTP requests begins much sooner than a WebSocket
   connection can be established, which more than makes up for the latency
   disadvantage.

5. HTTP requests are a little easier to inspect and debug in the dev tools
   than WebSocket frames.

6. The new /__dynamicImport HTTP endpoint is a raw Connect/Express-style
   endpoint, so it bypasses the Meteor method-calling system altogether,
   which eliminates some additional overhead.

7. Fetching dynamic modules no longer competes with other WebSocket
   traffic such as DDP and Livedata.

8. Although the implementation of the /__dynamicImport endpoint is a bit
   too complicated to allow serving dynamic modules from a CDN, that
   remains a possibility for future experimentation. In other words, how
   modules are fetched is still just an implementation detail of the
   `meteorInstall.fetch` callback.

9. As with the WebSocket implementation, the module server is totally
   stateless, so it should be easy to scale up if necessary.

I wish I had appreciated the advantages of an HTTP-based implementation
sooner, but I think the transition will be pretty seamless, since the new
implementation is completely backwards compatible with the old.

packages/dynamic-import/client.js

@@ -1,5 +1,7 @@
 var Module = module.constructor;
 var cache = require("./cache.js");
+var HTTP = require("meteor/http").HTTP;
+var meteorInstall = require("meteor/modules").meteorInstall;
 
 // Call module.dynamicImport(id) to fetch a module and any/all of its
 // dependencies that have not already been fetched, and evaluate them as
@@ -111,16 +113,12 @@ function makeModuleFunction(id, source, options) {
 }
 
 function fetchMissing(missingTree) {
-  // Update lastFetchMissingPromise immediately, without waiting for
-  // the results to be delivered.
   return new Promise(function (resolve, reject) {
-    Meteor.call(
-      "__dynamicImport",
-      missingTree,
-      function (error, resultsTree) {
-        error ? reject(error) : resolve(resultsTree);
-      }
-    );
+    HTTP.call("POST", Meteor.absoluteUrl("__dynamicImport"), {
+      data: missingTree
+    }, function (error, result) {
+      error ? reject(error) : resolve(result.data);
+    });
   });
 }
 

packages/dynamic-import/package.js

@@ -14,9 +14,8 @@ Package.onUse(function (api) {
 
   api.use("modules");
   api.use("promise");
-  api.use("ddp");
-  api.use("check", "server");
-  api.use("ecmascript", "server");
+  api.use("webapp", "server");
+  api.use("http");
 
   api.mainModule("client.js", "client");
   api.mainModule("server.js", "server");

packages/dynamic-import/server.js

@@ -1,17 +1,18 @@
-import assert from "assert";
-import { readFileSync } from "fs";
-import {
-  join as pathJoin,
-  normalize as pathNormalize,
-} from "path";
-
-import { check } from "meteor/check";
-
-import "./security.js";
-import "./client.js";
+"use strict";
 
+const assert = require("assert");
+const { readFileSync } = require("fs");
+const {
+  join: pathJoin,
+  normalize: pathNormalize,
+} = require("path");
 const hasOwn = Object.prototype.hasOwnProperty;
 
+const { WebApp } = require("meteor/webapp");
+
+require("./security.js");
+require("./client.js");
+
 Object.keys(dynamicImportInfo).forEach(platform => {
   const info = dynamicImportInfo[platform];
   if (info.dynamicRoot) {
@@ -19,30 +20,40 @@ Object.keys(dynamicImportInfo).forEach(platform => {
   }
 });
 
-Meteor.methods({
-  __dynamicImport(tree) {
-    check(tree, Object);
-    this.unblock();
-
-    const platform = this.connection ? "web.browser" : "server";
-    const pathParts = [];
-
-    function walk(node) {
-      if (node && typeof node === "object") {
-        Object.keys(node).forEach(name => {
-          pathParts.push(name);
-          node[name] = walk(node[name]);
-          assert.strictEqual(pathParts.pop(), name);
-        });
-      } else {
-        return read(pathParts, platform);
-      }
-      return node;
-    }
-
-    return walk(tree);
+WebApp.connectHandlers.use(
+  "/__dynamicImport",
+  function (request, response) {
+    assert.strictEqual(request.method, "POST");
+    const chunks = [];
+    request.on("data", chunk => chunks.push(chunk));
+    request.on("end", () => {
+      response.setHeader("Content-Type", "application/json");
+      response.end(JSON.stringify(readTree(
+        JSON.parse(Buffer.concat(chunks)),
+        "web.browser"
+      )));
+    });
   }
-});
+);
+
+function readTree(tree, platform) {
+  const pathParts = [];
+
+  function walk(node) {
+    if (node && typeof node === "object") {
+      Object.keys(node).forEach(name => {
+        pathParts.push(name);
+        node[name] = walk(node[name]);
+        assert.strictEqual(pathParts.pop(), name);
+      });
+    } else {
+      return read(pathParts, platform);
+    }
+    return node;
+  }
+
+  return walk(tree);
+}
 
 function read(pathParts, platform) {
   const { dynamicRoot } = dynamicImportInfo[platform];