From dac580e2122dcf843c4b241da8d7d9809cf537d5 Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Wed, 23 Dec 2015 15:14:31 +0100 Subject: [PATCH 1/9] Set ETag header to asset hash if available Previously, the ETag header was set (by `send`) to a default value based on the inode of the file. Using the asset hash instead allows for proper conditional requests even after redeployments. To take advantage of content-based caching, we also have to disable the Last-Modified header because having this set to the file date would still make requests conditional on the most recent deployment. This requires updating the send dependency and is done in a separate commit. Fixes #626. --- packages/webapp/webapp_server.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index d3ef354378..7882c86ecb 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -403,6 +403,10 @@ WebAppInternals.staticFilesMiddleware = function (staticFiles, req, res, next) { } } + if (info.hash) { + res.setHeader('ETag', info.hash); + } + if (info.content) { res.write(info.content); res.end(); @@ -475,6 +479,7 @@ var runWebAppServer = function () { staticFiles[urlPrefix + getItemPathname(item.url)] = { absolutePath: path.join(clientDir, item.path), cacheable: item.cacheable, + hash: item.hash, // Link from source to its map sourceMapUrl: item.sourceMapUrl, type: item.type From 5e9090c3ee86c21541a5bc8e4ec9bcde81251977 Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Wed, 23 Dec 2015 15:20:45 +0100 Subject: [PATCH 2/9] Use location.replace() instead of location.reload() to avoid unnecessary asset validation If a cached version of an asset is still fresh (depending on the `max-age`), there is no need to send even a conditional request. Because `location.reload()` for some reason does force validation, it makes sense to use `location.replace` instead and improve reloading performance by avoiding unnecessary requests. --- packages/reload/reload.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/reload/reload.js b/packages/reload/reload.js index 2a400bdece..86a841883a 100644 --- a/packages/reload/reload.js +++ b/packages/reload/reload.js @@ -218,8 +218,10 @@ Reload._reload = function (options) { var tryReload = function () { _.defer(function () { if (Reload._migrate(tryReload, options)) { - // Tell the browser to shut down this VM and make a new one - window.location.reload(); + // Make the browser reload the page + // Using location.replace() instead of location.reload() avoids + // validating assets with the server if we still have a fresh cached copy. + window.location.replace(window.location.href); } }); }; From 5d501d17936de234fa0e992e590f70bf958e98db Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Thu, 7 Jan 2016 13:26:47 +0100 Subject: [PATCH 3/9] Update send and connect dependencies for webapp package Updating send was necessary to allow disabling setting Last-Modified, and version dependencies required updating connect as well. --- .../webapp/.npm/package/npm-shrinkwrap.json | 406 ++++++++++++++++-- packages/webapp/package.js | 4 +- packages/webapp/webapp_server.js | 8 +- 3 files changed, 370 insertions(+), 48 deletions(-) diff --git a/packages/webapp/.npm/package/npm-shrinkwrap.json b/packages/webapp/.npm/package/npm-shrinkwrap.json index dc8dc414d6..2bc74cdd05 100644 --- a/packages/webapp/.npm/package/npm-shrinkwrap.json +++ b/packages/webapp/.npm/package/npm-shrinkwrap.json @@ -1,65 +1,387 @@ { "dependencies": { "connect": { - "version": "2.9.0", + "version": "2.30.2", "dependencies": { - "qs": { - "version": "0.6.5" + "basic-auth-connect": { + "version": "1.0.0" }, - "cookie-signature": { - "version": "1.0.1" - }, - "buffer-crc32": { - "version": "0.2.1" - }, - "cookie": { - "version": "0.1.0" - }, - "bytes": { - "version": "0.2.0" - }, - "fresh": { - "version": "0.2.0" - }, - "pause": { - "version": "0.0.1" - }, - "uid2": { - "version": "0.0.2" - }, - "debug": { - "version": "0.7.2" - }, - "methods": { - "version": "0.0.1" - }, - "multiparty": { - "version": "2.1.8", + "body-parser": { + "version": "1.13.3", "dependencies": { - "readable-stream": { - "version": "1.0.17" + "iconv-lite": { + "version": "0.4.11" }, - "stream-counter": { - "version": "0.1.0" + "on-finished": { + "version": "2.3.0", + "dependencies": { + "ee-first": { + "version": "1.1.1" + } + } + }, + "raw-body": { + "version": "2.1.5", + "dependencies": { + "bytes": { + "version": "2.2.0" + }, + "iconv-lite": { + "version": "0.4.13" + }, + "unpipe": { + "version": "1.0.0" + } + } } } + }, + "bytes": { + "version": "2.1.0" + }, + "cookie": { + "version": "0.1.3" + }, + "cookie-parser": { + "version": "1.3.5" + }, + "cookie-signature": { + "version": "1.0.6" + }, + "compression": { + "version": "1.5.2", + "dependencies": { + "accepts": { + "version": "1.2.13", + "dependencies": { + "mime-types": { + "version": "2.1.9", + "dependencies": { + "mime-db": { + "version": "1.21.0" + } + } + }, + "negotiator": { + "version": "0.5.3" + } + } + }, + "compressible": { + "version": "2.0.6", + "dependencies": { + "mime-db": { + "version": "1.21.0" + } + } + }, + "vary": { + "version": "1.0.1" + } + } + }, + "connect-timeout": { + "version": "1.6.2", + "dependencies": { + "ms": { + "version": "0.7.1" + } + } + }, + "content-type": { + "version": "1.0.1" + }, + "csurf": { + "version": "1.8.3", + "dependencies": { + "csrf": { + "version": "3.0.0", + "dependencies": { + "base64-url": { + "version": "1.2.1" + }, + "rndm": { + "version": "1.1.1" + }, + "scmp": { + "version": "1.0.0" + }, + "uid-safe": { + "version": "2.0.0" + } + } + } + } + }, + "debug": { + "version": "2.2.0", + "dependencies": { + "ms": { + "version": "0.7.1" + } + } + }, + "depd": { + "version": "1.0.1" + }, + "errorhandler": { + "version": "1.4.2", + "dependencies": { + "accepts": { + "version": "1.2.13", + "dependencies": { + "mime-types": { + "version": "2.1.9", + "dependencies": { + "mime-db": { + "version": "1.21.0" + } + } + }, + "negotiator": { + "version": "0.5.3" + } + } + }, + "escape-html": { + "version": "1.0.2" + } + } + }, + "express-session": { + "version": "1.11.3", + "dependencies": { + "crc": { + "version": "3.3.0" + }, + "uid-safe": { + "version": "2.0.0", + "dependencies": { + "base64-url": { + "version": "1.2.1" + } + } + } + } + }, + "finalhandler": { + "version": "0.4.0", + "dependencies": { + "escape-html": { + "version": "1.0.2" + }, + "on-finished": { + "version": "2.3.0", + "dependencies": { + "ee-first": { + "version": "1.1.1" + } + } + }, + "unpipe": { + "version": "1.0.0" + } + } + }, + "fresh": { + "version": "0.3.0" + }, + "http-errors": { + "version": "1.3.1", + "dependencies": { + "inherits": { + "version": "2.0.1" + }, + "statuses": { + "version": "1.2.1" + } + } + }, + "method-override": { + "version": "2.3.5", + "dependencies": { + "methods": { + "version": "1.1.1" + }, + "vary": { + "version": "1.0.1" + } + } + }, + "morgan": { + "version": "1.6.1", + "dependencies": { + "basic-auth": { + "version": "1.0.3" + }, + "on-finished": { + "version": "2.3.0", + "dependencies": { + "ee-first": { + "version": "1.1.1" + } + } + } + } + }, + "multiparty": { + "version": "3.3.2", + "dependencies": { + "readable-stream": { + "version": "1.1.13", + "dependencies": { + "core-util-is": { + "version": "1.0.2" + }, + "isarray": { + "version": "0.0.1" + }, + "string_decoder": { + "version": "0.10.31" + }, + "inherits": { + "version": "2.0.1" + } + } + }, + "stream-counter": { + "version": "0.2.0" + } + } + }, + "on-headers": { + "version": "1.0.1" + }, + "parseurl": { + "version": "1.3.0" + }, + "pause": { + "version": "0.1.0" + }, + "qs": { + "version": "4.0.0" + }, + "response-time": { + "version": "2.3.1" + }, + "serve-favicon": { + "version": "2.3.0", + "dependencies": { + "etag": { + "version": "1.7.0" + }, + "ms": { + "version": "0.7.1" + } + } + }, + "serve-index": { + "version": "1.7.2", + "dependencies": { + "accepts": { + "version": "1.2.13", + "dependencies": { + "negotiator": { + "version": "0.5.3" + } + } + }, + "batch": { + "version": "0.5.2" + }, + "escape-html": { + "version": "1.0.2" + }, + "mime-types": { + "version": "2.1.9", + "dependencies": { + "mime-db": { + "version": "1.21.0" + } + } + } + } + }, + "serve-static": { + "version": "1.10.0", + "dependencies": { + "escape-html": { + "version": "1.0.2" + } + } + }, + "type-is": { + "version": "1.6.10", + "dependencies": { + "media-typer": { + "version": "0.3.0" + }, + "mime-types": { + "version": "2.1.9", + "dependencies": { + "mime-db": { + "version": "1.21.0" + } + } + } + } + }, + "utils-merge": { + "version": "1.0.0" + }, + "vhost": { + "version": "3.0.2" } } }, "send": { - "version": "0.1.4", + "version": "0.13.0", "dependencies": { "debug": { - "version": "0.7.2" + "version": "2.2.0" }, - "mime": { - "version": "1.2.11" + "depd": { + "version": "1.0.1" + }, + "destroy": { + "version": "1.0.3" + }, + "escape-html": { + "version": "1.0.2" + }, + "etag": { + "version": "1.7.0" }, "fresh": { - "version": "0.2.0" + "version": "0.3.0" + }, + "http-errors": { + "version": "1.3.1", + "dependencies": { + "inherits": { + "version": "2.0.1" + } + } + }, + "mime": { + "version": "1.3.4" + }, + "ms": { + "version": "0.7.1" + }, + "on-finished": { + "version": "2.3.0", + "dependencies": { + "ee-first": { + "version": "1.1.1" + } + } }, "range-parser": { - "version": "0.0.4" + "version": "1.0.3" + }, + "statuses": { + "version": "1.2.1" } } }, diff --git a/packages/webapp/package.js b/packages/webapp/package.js index 68ff08e369..5990364a2e 100644 --- a/packages/webapp/package.js +++ b/packages/webapp/package.js @@ -3,8 +3,8 @@ Package.describe({ version: '1.2.3' }); -Npm.depends({connect: "2.9.0", - send: "0.1.4", +Npm.depends({connect: "2.30.2", + send: "0.13.0", useragent: "2.0.7"}); Npm.strip({ diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index 7882c86ecb..3270230f69 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -411,10 +411,10 @@ WebAppInternals.staticFilesMiddleware = function (staticFiles, req, res, next) { res.write(info.content); res.end(); } else { - send(req, info.absolutePath) - .maxage(maxAge) - .hidden(true) // if we specified a dotfile in the manifest, serve it - .on('error', function (err) { + send(req, info.absolutePath, { + maxage: maxAge, + dotfiles: 'allow' // if we specified a dotfile in the manifest, serve it + }).on('error', function (err) { Log.error("Error serving static file " + err); res.writeHead(500); res.end(); From b8a17dd97f7a6972f03f00ea848e7c149c52cab1 Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Thu, 7 Jan 2016 13:27:55 +0100 Subject: [PATCH 4/9] Serve non-cacheable files with a max-age of 0 instead of a day We previously served non-cacheable files with a max-age of a day. This was done to avoid image flickering on page reload (see #773). As far as I can tell, image flickering still occurs because `location.reload` always forces validation. But switching to `location.replace` means that max-age will actually be respected, and we don't want to cache these assets for a day because then changes would not be visible on reloads. --- packages/webapp/webapp_server.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index 3270230f69..6130237587 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -367,17 +367,9 @@ WebAppInternals.staticFilesMiddleware = function (staticFiles, req, res, next) { // Cacheable files are files that should never change. Typically // named by their hash (eg meteor bundled js and css files). // We cache them ~forever (1yr). - // - // We cache non-cacheable files anyway. This isn't really correct, as users - // can change the files and changes won't propagate immediately. However, if - // we don't cache them, browsers will 'flicker' when rerendering - // images. Eventually we will probably want to rewrite URLs of static assets - // to include a query parameter to bust caches. That way we can both get - // good caching behavior and allow users to change assets without delay. - // https://github.com/meteor/meteor/issues/773 var maxAge = info.cacheable ? 1000 * 60 * 60 * 24 * 365 - : 1000 * 60 * 60 * 24; + : 0; // Set the X-SourceMap header, which current Chrome, FireFox, and Safari // understand. (The SourceMap header is slightly more spec-correct but FF From 0a31dabcd62f7d55cc208a0520c9ea433c6ff2ec Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Thu, 7 Jan 2016 13:56:45 +0100 Subject: [PATCH 5/9] Don't set Last-Modified based on the file date If we serve files with a Last-Modified based on the file date, this interferes with content-based cache validation using ETag because clients are required to take both into account. --- packages/webapp/webapp_server.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index 6130237587..1e8b78d18d 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -405,7 +405,8 @@ WebAppInternals.staticFilesMiddleware = function (staticFiles, req, res, next) { } else { send(req, info.absolutePath, { maxage: maxAge, - dotfiles: 'allow' // if we specified a dotfile in the manifest, serve it + dotfiles: 'allow', // if we specified a dotfile in the manifest, serve it + lastModified: false // don't set last-modified based on the file date }).on('error', function (err) { Log.error("Error serving static file " + err); res.writeHead(500); From 21ee605541e864df2cf9dcf82802a4166e34519f Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Thu, 7 Jan 2016 14:36:49 +0100 Subject: [PATCH 6/9] Support query parameter to avoid downloading index page when a file does not exist When downloading files during a Cordova hot code push, we need to detect if a file is not available instead of inadvertently downloading the default index page. --- packages/webapp/webapp_server.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index 1e8b78d18d..29a9b28bff 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -685,6 +685,17 @@ var runWebAppServer = function () { return undefined; } + if (request.url.query && request.url.query['meteor_dont_serve_index']) { + // When downloading files during a Cordova hot code push, we need + // to detect if a file is not available instead of inadvertently + // downloading the default index page. + // So similar to the situation above, we serve an uncached 404. + headers['Cache-Control'] = 'no-cache'; + res.writeHead(404, headers); + res.end("404 Not Found"); + return undefined; + } + // /packages/asdfsad ... /__cordova/dafsdf.js var pathname = connect.utils.parseUrl(req).pathname; var archKey = pathname.split('/')[1]; From fdebebb7ed71968da23ff833ea16b6453157b495 Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Fri, 8 Jan 2016 11:09:58 +0100 Subject: [PATCH 7/9] Replace deprecated connect.utils.parseUrl() with parseurl module As recommended in the deprecation warning, we replace connect.utils.parseUrl() usage with using the parseurl module directly. --- packages/webapp/.npm/package/npm-shrinkwrap.json | 3 +++ packages/webapp/package.js | 1 + packages/webapp/webapp_server.js | 5 +++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/webapp/.npm/package/npm-shrinkwrap.json b/packages/webapp/.npm/package/npm-shrinkwrap.json index 2bc74cdd05..ce2b3a2da8 100644 --- a/packages/webapp/.npm/package/npm-shrinkwrap.json +++ b/packages/webapp/.npm/package/npm-shrinkwrap.json @@ -334,6 +334,9 @@ } } }, + "parseurl": { + "version": "1.3.0" + }, "send": { "version": "0.13.0", "dependencies": { diff --git a/packages/webapp/package.js b/packages/webapp/package.js index 5990364a2e..06fc1e0ecb 100644 --- a/packages/webapp/package.js +++ b/packages/webapp/package.js @@ -4,6 +4,7 @@ Package.describe({ }); Npm.depends({connect: "2.30.2", + parseurl: "1.3.0", send: "0.13.0", useragent: "2.0.7"}); diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index 29a9b28bff..9d9e410f25 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -8,6 +8,7 @@ var url = Npm.require("url"); var crypto = Npm.require("crypto"); var connect = Npm.require('connect'); +var parseurl = Npm.require('parseurl'); var useragent = Npm.require('useragent'); var send = Npm.require('send'); @@ -326,7 +327,7 @@ WebAppInternals.staticFilesMiddleware = function (staticFiles, req, res, next) { next(); return; } - var pathname = connect.utils.parseUrl(req).pathname; + var pathname = parseurl(req).pathname; try { pathname = decodeURIComponent(pathname); } catch (e) { @@ -697,7 +698,7 @@ var runWebAppServer = function () { } // /packages/asdfsad ... /__cordova/dafsdf.js - var pathname = connect.utils.parseUrl(req).pathname; + var pathname = parseurl(req).pathname; var archKey = pathname.split('/')[1]; var archKeyCleaned = 'web.' + archKey.replace(/^__/, ''); From ce628dfb814b8ed725d09bdfc49706828f963a9f Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Mon, 11 Jan 2016 11:14:36 +0100 Subject: [PATCH 8/9] ETags should be enclosed in double quotes --- packages/webapp/webapp_server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index 9d9e410f25..5908864fae 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -397,7 +397,7 @@ WebAppInternals.staticFilesMiddleware = function (staticFiles, req, res, next) { } if (info.hash) { - res.setHeader('ETag', info.hash); + res.setHeader('ETag', '"' + info.hash + '"'); } if (info.content) { From bdbdad91d3a1a7aa38842f0fde2fed3ec9ca396f Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Mon, 11 Jan 2016 12:50:54 +0100 Subject: [PATCH 9/9] Set hash to version when serving manifest.json --- packages/webapp/webapp_server.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index 5908864fae..961e23fafb 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -504,6 +504,7 @@ var runWebAppServer = function () { staticFiles[path.join(urlPrefix, getItemPathname('/manifest.json'))] = { content: JSON.stringify(program), cacheable: true, + hash: program.version, type: "json" }; };