diff --git a/History.md b/History.md index 27998afb53..ef340a8a32 100644 --- a/History.md +++ b/History.md @@ -1,6 +1,10 @@ ## vNEXT +* Add new `appcache` package. Add this package to your project to speed + up page load and make hot code reload smoother using the HTML5 + AppCache API. See http://docs.meteor.com/#appcache for details. + * You can now provide a `transform` option to collections, which is a function that documents coming out of that collection are passed through. `find`, `findOne`, `allow`, and `deny` now take `transform` options, diff --git a/app/lib/bundler.js b/app/lib/bundler.js index 2bd7b98146..2c172f89c6 100644 --- a/app/lib/bundler.js +++ b/app/lib/bundler.js @@ -12,7 +12,7 @@ // - manifest [list of resources in load order, each consists of an object]: // { // "path": relative path of file in the bundle, normalized to use forward slashes -// "where": "client" [could also be "server" in future] +// "where": "client", "internal" [could also be "server" in future] // "type": "js", "css", or "static" // "cacheable": (client) boolean, is it safe to ask the browser to cache this file // "url": (client) relative url to download the resource, includes cache @@ -231,11 +231,19 @@ var Bundle = function () { // list of filenames self.css = []; + // images and other static files added from packages + // map from environment, to list of filenames + self.static = {client: [], server: []}; + // Map from environment, to path name (server relative), to contents // of file as buffer. self.files = {client: {}, client_cacheable: {}, server: {}}; - // See description of manifest at the top + // See description of the manifest at the top. + // Note that in contrast to self.js etc., the manifest only includes + // files which are in the final bundler output: for example, if code + // is minified, the manifest includes the minify output file but not + // the individual input files that were combined. self.manifest = []; // list of segments of additional HTML for / @@ -315,6 +323,7 @@ var Bundle = function () { self[options.type].push(data); } else if (options.type === "static") { self.files[w][options.path] = data; + self.static[w].push(options.path); } else { throw new Error("Unknown type " + options.type); } @@ -529,6 +538,25 @@ _.extend(Bundle.prototype, { // --- Static assets --- + var addClientFileToManifest = function (filepath, contents, type, cacheable, url) { + if (! contents instanceof Buffer) + throw new Error('contents must be a Buffer'); + var normalized = filepath.split(path.sep).join('/'); + if (normalized.charAt(0) === '/') + normalized = normalized.substr(1); + self.manifest.push({ + // path is normalized to use forward slashes + path: (cacheable ? 'static_cacheable' : 'static') + '/' + normalized, + where: 'client', + type: type, + cacheable: cacheable, + url: url || '/' + normalized, + // contents is a Buffer and so correctly gives us the size in bytes + size: contents.length, + hash: self._hash(contents) + }); + }; + if (is_app) { if (fs.existsSync(path.join(project_dir, 'public'))) { var copied = @@ -537,18 +565,8 @@ _.extend(Bundle.prototype, { _.each(copied, function (fs_relative_path) { var filepath = path.join(build_path, 'static', fs_relative_path); - var normalized = fs_relative_path.split(path.sep).join('/'); - self.manifest.push({ - // path is normalized to use forward slashes, so deliberately - // not using path.sep here - path: 'static/' + normalized, - type: 'static', - where: 'client', - cacheable: false, - url: '/' + normalized, - size: fs.statSync(filepath).size, - hash: self._hash(fs.readFileSync(filepath)) - }); + var contents = fs.readFileSync(filepath); + addClientFileToManifest(fs_relative_path, contents, 'static', false); }); } dependencies_json.app.push('public'); @@ -573,17 +591,7 @@ _.extend(Bundle.prototype, { else throw new Error('unable to find file: ' + file); - self.manifest.push({ - // path is normalized to use forward slashes - path: 'static_cacheable' + file.split(path.sep).join('/'), - where: 'client', - type: type, - cacheable: true, - url: url, - // contents is a Buffer and so correctly gives us the size in bytes - size: contents.length, - hash: self._hash(contents) - }); + addClientFileToManifest(file, contents, type, true, url); }; _.each(self.js.client, function (file) { processClientCode('js', file); }); @@ -594,6 +602,7 @@ _.extend(Bundle.prototype, { var full_path = path.join(build_path, 'static', rel_path); files.mkdir_p(path.dirname(full_path), 0755); fs.writeFileSync(full_path, self.files.client[rel_path]); + addClientFileToManifest(rel_path, self.files.client[rel_path], 'static', false); } // -- Client cache forever code -- @@ -613,8 +622,13 @@ _.extend(Bundle.prototype, { fs.writeFileSync(full_path, self.files.server[rel_path]); } - fs.writeFileSync(path.join(build_path, 'app.html'), - self._generate_app_html()); + var app_html = self._generate_app_html(); + fs.writeFileSync(path.join(build_path, 'app.html'), app_html); + self.manifest.push({ + path: 'app.html', + where: 'internal', + hash: self._hash(app_html) + }); dependencies_json.core.push(path.join('lib', 'app.html.in')); fs.writeFileSync(path.join(build_path, 'unsupported.html'), diff --git a/app/server/server.js b/app/server/server.js index 69586bdd10..8923ff26af 100644 --- a/app/server/server.js +++ b/app/server/server.js @@ -155,9 +155,9 @@ var appUrl = function (url) { if (url === '/app.manifest') return false; - // Avoid serving app HTML for declared network routes such as /sockjs/. + // Avoid serving app HTML for declared routes such as /sockjs/. if (__meteor_bootstrap__._routePolicy && - __meteor_bootstrap__._routePolicy.classify(url) === 'network') + __meteor_bootstrap__._routePolicy.classify(url)) return false; // we currently return app HTML on all URLs by default diff --git a/docs/.meteor/packages b/docs/.meteor/packages index 36eb4b5aea..cfb17d208a 100644 --- a/docs/.meteor/packages +++ b/docs/.meteor/packages @@ -10,3 +10,4 @@ code-prettify jquery-waypoints less spiderable +appcache diff --git a/docs/client/docs.js b/docs/client/docs.js index 19e15783d3..f66b34bb3a 100644 --- a/docs/client/docs.js +++ b/docs/client/docs.js @@ -291,6 +291,7 @@ var toc = [ "Packages", [ [ "accounts-ui", + "appcache", "amplify", "backbone", "bootstrap", diff --git a/docs/client/introduction.html b/docs/client/introduction.html index b48f876cad..82e53b0f7b 100644 --- a/docs/client/introduction.html +++ b/docs/client/introduction.html @@ -107,7 +107,7 @@ clean, classically beautiful APIs.

Developer Resources

-Fork me on GitHub +Fork me on GitHub If anything in Meteor catches your interest, we hope you'll get involved with the project! diff --git a/docs/client/packages.html b/docs/client/packages.html index adfd08f2b8..0d2d9e892e 100644 --- a/docs/client/packages.html +++ b/docs/client/packages.html @@ -17,6 +17,7 @@ and removed with: $ meteor remove {{> pkg_accounts_ui}} +{{> pkg_appcache}} {{> pkg_amplify}} {{> pkg_backbone}} {{> pkg_bootstrap}} diff --git a/docs/client/packages/appcache.html b/docs/client/packages/appcache.html new file mode 100644 index 0000000000..35e94c8902 --- /dev/null +++ b/docs/client/packages/appcache.html @@ -0,0 +1,92 @@ + diff --git a/docs/public/forkme_right_red_aa0000.png b/docs/public/forkme_right_red_aa0000.png new file mode 100644 index 0000000000..1e19c21262 Binary files /dev/null and b/docs/public/forkme_right_red_aa0000.png differ diff --git a/packages/appcache/QA.md b/packages/appcache/QA.md new file mode 100644 index 0000000000..b30aa23d0f --- /dev/null +++ b/packages/appcache/QA.md @@ -0,0 +1,93 @@ +# QA Notes + +## Viewing the app cache + +Chrome: Navigate to chrome://appcache-internals/ + +Firefox: Open Tools / Advanced / Network. The section reading "The +following websites are allowed to store data for offline use" will +show the amount of data in the app cache ("1.2 MB"). If this number +is 0 the app is permitted to use the app cache but the app cache is +currently turned off. + + +## Setup + +Create a simple static app and add the appcache package. + +static.html: + +```` + + some static content + +```` + +If you're testing with Firefox, enable it: + +static.js: + +```` +if (Meteor.isServer) { + Meteor.AppCache.config({ + firefox: true + }); +} +```` + + +## App is cached offline + +Run Meteor, load the app in the browser, stop Meteor. Reload the page +in the browser and observe the content is still visible. + + +## Hot code reload still works + +Run Meteor, open the app in the browser. Make a change to +static.html. Observe the change appear in the web page. + +Note that it is normal when using the app cache for the page reload to +be delayed a bit while the browser fetches the changed code in the +background. + +Without app cache: (page goes blank) -> (browser fetches) -> (page renders) + +With app cache: (browser fetches) -> (page goes blank) -> (page renders) + + +## Enabling / disabling the appcache turns the app cache on / off + +Run Meteor, open the app in the browser. + +Disable your browser in the appcache config. For example, if you're +using Chrome: + +```` +Meteor.AppCache.config({ + chrome: false +}); +```` + +Observe following the hot code reload the app is no longer cached. + +Enable your browser again: + +```` +Meteor.AppCache.config({ + chrome: true +}); +```` + +Observe following the hot code reload the app is cached again. + + +## Removing the appcache package turns off app caching + +Start Meteor, open the app in the browser. + +Stop Meteor, remove the appcache package, remove or comment out the +call to Meteor.AppCache.config in static.js, start Meteor again. + +Wait for the browser to reestablish its livedata connection. Observe +following the hot code reload that the app is no longer cached. diff --git a/packages/appcache/appcache-client.js b/packages/appcache/appcache-client.js new file mode 100644 index 0000000000..cae65a397e --- /dev/null +++ b/packages/appcache/appcache-client.js @@ -0,0 +1,71 @@ +(function() { + + if (! window.applicationCache) + return; + + var appCacheStatuses = [ + 'uncached', + 'idle', + 'checking', + 'downloading', + 'updateready', + 'obsolete' + ]; + + var updatingAppcache = false; + var reloadRetry = null; + var appcacheUpdated = false; + + Meteor._reload.onMigrate('appcache', function(retry) { + if (appcacheUpdated) + return [true]; + + // An uncached application (one that does not have a manifest) cannot + // be updated. + if (window.applicationCache.status === window.applicationCache.UNCACHED) + return [true]; + + if (!updatingAppcache) { + try { + window.applicationCache.update(); + } catch (e) { + Meteor._debug('applicationCache update error', e); + // There's no point in delaying the reload if we can't update the cache. + return [true]; + } + updatingAppcache = true; + } + + // Delay migration until the app cache has been updated. + reloadRetry = retry; + return false; + }); + + // If we're migrating and the app cache is now up to date, signal that + // we're now ready to migrate. + var cacheIsNowUpToDate = function() { + if (!updatingAppcache) + return; + appcacheUpdated = true; + reloadRetry(); + }; + + window.applicationCache.addEventListener('updateready', cacheIsNowUpToDate, false); + window.applicationCache.addEventListener('noupdate', cacheIsNowUpToDate, false); + + // We'll get the obsolete event on a 404 fetching the app.manifest: + // we had previously been running with an app cache, but the app + // cache has now been disabled or the appcache package removed. + // Reload to get the new non-cached code. + + window.applicationCache.addEventListener('obsolete', (function() { + if (reloadRetry) { + cacheIsNowUpToDate(); + } + else { + appcacheUpdated = true; + Meteor._reload.reload(); + } + }), false); + +})(); diff --git a/packages/appcache/appcache-server.js b/packages/appcache/appcache-server.js new file mode 100644 index 0000000000..a0b5badd4d --- /dev/null +++ b/packages/appcache/appcache-server.js @@ -0,0 +1,185 @@ +(function() { + + var app = __meteor_bootstrap__.app; + var bundle = __meteor_bootstrap__.bundle; + var crypto = __meteor_bootstrap__.require('crypto'); + var fs = __meteor_bootstrap__.require('fs'); + var path = __meteor_bootstrap__.require('path'); + + var knownBrowsers = ['android', 'chrome', 'firefox', 'ie', 'mobileSafari', 'safari']; + + var browsersEnabledByDefault = ['android', 'chrome', 'ie', 'mobileSafari', 'safari']; + + var enabledBrowsers = {}; + _.each(browsersEnabledByDefault, function (browser) { + enabledBrowsers[browser] = true; + }); + + Meteor.AppCache = { + config: function(options) { + _.each(options, function (value, option) { + if (option === 'browsers') { + enabledBrowsers = {}; + _.each(value, function (browser) { + enabledBrowsers[browser] = true; + }); + } + else if (_.contains(knownBrowsers, option)) { + enabledBrowsers[option] = value; + } + else if (option === 'onlineOnly') { + _.each(value, function (urlPrefix) { + Meteor._routePolicy.declare(urlPrefix, 'static-online'); + }); + } + else { + throw new Error('Invalid AppCache config option: ' + option); + } + }); + } + }; + + var browserEnabled = function(request) { + return enabledBrowsers[request.browser.name]; + }; + + __meteor_bootstrap__.htmlAttributeHooks.push(function (request) { + if (browserEnabled(request)) + return 'manifest="/app.manifest"'; + else + return null; + }); + + app.use(function(req, res, next) { + if (req.url !== '/app.manifest') { + return next(); + } + + // Browsers will get confused if we unconditionally serve the + // manifest and then disable the app cache for that browser. If + // the app cache had previously been enabled for a browser, it + // will continue to fetch the manifest as long as it's available, + // even if we now are not including the manifest attribute in the + // app HTML. (Firefox for example will continue to display "this + // website is asking to store data on your computer for offline + // use"). Returning a 404 gets the browser to really turn off the + // app cache. + + if (!browserEnabled(__meteor_bootstrap__.categorizeRequest(req))) { + res.writeHead(404); + res.end(); + return; + } + + // After the browser has downloaded the app files from the server and + // has populated the browser's application cache, the browser will + // *only* connect to the server and reload the application if the + // *contents* of the app manifest file has changed. + // + // So we have to ensure that if any static client resources change, + // something changes in the manifest file. We compute a hash of + // everything that gets delivered to the client during the initial + // web page load, and include that hash as a comment in the app + // manifest. That way if anything changes, the comment changes, and + // the browser will reload resources. + + var hash = crypto.createHash('sha1'); + hash.update(JSON.stringify(__meteor_runtime_config__), 'utf8'); + _.each(bundle.manifest, function (resource) { + if (resource.where === 'client' || resource.where === 'internal') { + hash.update(resource.hash); + } + }); + var digest = hash.digest('hex'); + + var manifest = "CACHE MANIFEST\n\n"; + manifest += '# ' + digest + "\n\n"; + + manifest += "CACHE:" + "\n"; + manifest += "/" + "\n"; + _.each(bundle.manifest, function (resource) { + if (resource.where === 'client' && + ! Meteor._routePolicy.classify(resource.url)) { + manifest += resource.url; + // If the resource is not already cacheable (has a query + // parameter, presumably with a hash or version of some sort), + // put a version with a hash in the cache. + // + // Avoid putting a non-cacheable asset into the cache, otherwise + // the user can't modify the asset until the cache headers + // expire. + if (!resource.cacheable) + manifest += "?" + resource.hash; + + manifest += "\n"; + } + }); + manifest += "\n"; + + manifest += "FALLBACK:\n"; + manifest += "/ /" + "\n"; + // Add a fallback entry for each uncacheable asset we added above. + // + // This means requests for the bare url (/image.png instead of + // /image.png?hash) will work offline. Online, however, the browser + // will send a request to the server. Users can remove this extra + // request to the server and have the asset served from cache by + // specifying the full URL with hash in their code (manually, with + // some sort of URL rewriting helper) + _.each(bundle.manifest, function (resource) { + if (resource.where === 'client' && + ! Meteor._routePolicy.classify(resource.url) && + !resource.cacheable) { + manifest += resource.url + " " + resource.url + + "?" + resource.hash + "\n"; + } + }); + + manifest += "\n"; + + manifest += "NETWORK:\n"; + // TODO adding the manifest file to NETWORK should be unnecessary? + // Want more testing to be sure. + manifest += "/app.manifest" + "\n"; + _.each( + [].concat( + Meteor._routePolicy.urlPrefixesFor('network'), + Meteor._routePolicy.urlPrefixesFor('static-online') + ), + function (urlPrefix) { + manifest += urlPrefix + "\n"; + } + ); + manifest += "*" + "\n"; + + // content length needs to be based on bytes + var body = new Buffer(manifest); + + res.setHeader('Content-Type', 'text/cache-manifest'); + res.setHeader('Content-Length', body.length); + return res.end(body); + }); + + var sizeCheck = function() { + var totalSize = 0; + _.each(bundle.manifest, function (resource) { + if (resource.where === 'client') { + totalSize += resource.size; + } + }); + if (totalSize > 5 * 1024 * 1024) { + Meteor._debug( + "** You are using the appcache package but the total size of the\n" + + "** cached resources is " + + (totalSize / 1024 / 1024).toFixed(1) + "MB.\n" + + "**\n" + + "** This is over the recommended maximum of 5 MB and may break your\n" + + "** app in some browsers! See http://docs.meteor.com/#appcache\n" + + "** for more information and fixes.\n" + ); + } + }; + + sizeCheck(); + +})(); diff --git a/packages/appcache/package.js b/packages/appcache/package.js new file mode 100644 index 0000000000..53f4526877 --- /dev/null +++ b/packages/appcache/package.js @@ -0,0 +1,11 @@ +Package.describe({ + summary: "enable the application cache in the browser" +}); + +Package.on_use(function (api) { + api.use('reload', 'client'); + api.use('routepolicy', 'server'); + api.use('startup', 'client'); + api.add_files('appcache-client.js', 'client'); + api.add_files('appcache-server.js', 'server'); +}); diff --git a/packages/routepolicy/routepolicy.js b/packages/routepolicy/routepolicy.js index ce27701568..72300fa479 100644 --- a/packages/routepolicy/routepolicy.js +++ b/packages/routepolicy/routepolicy.js @@ -1,3 +1,25 @@ +// In addition to listing specific files to be cached, the browser +// application cache manifest allows URLs to be designated as NETWORK +// (always fetched from the Internet) and FALLBACK (which we use to +// serve app HTML on arbitrary URLs). +// +// The limitation of the manifest file format is that the designations +// are by prefix only: if "/foo" is declared NETWORK then "/foobar" +// will also be treated as a network route. +// +// Meteor._routePolicy is a low-level API for declaring the route type +// of URL prefixes: +// +// "network": for network routes that should not conflict with static +// resources. (For example, if "/sockjs/" is a network route, we +// shouldn't have "/sockjs/red-sock.jpg" as a static resource). +// +// "static-online": for static resources which should not be cached in +// the app cache. This is implemented by also adding them to the +// NETWORK section (as otherwise the browser would receive app HTML +// for them because of the FALLBACK section), but static-online routes +// don't need to be checked for conflict with static resources. + (function () { // The route policy is a singleton in a running application, but we @@ -17,8 +39,8 @@ }, checkType: function (type) { - if (! _.contains(['network'], type)) - return 'the route type must be "network"'; + if (! _.contains(['network', 'static-online'], type)) + return 'the route type must be "network" or "static-online"'; return null; }, @@ -35,6 +57,8 @@ checkForConflictWithStatic: function (urlPrefix, type, _testManifest) { var self = this; + if (type === 'static-online') + return null; var manifest = _testManifest || __meteor_bootstrap__.bundle.manifest; var conflict = _.find(manifest, function (resource) { return (resource.type === 'static' && diff --git a/packages/routepolicy/routepolicy_tests.js b/packages/routepolicy/routepolicy_tests.js index 51c38eaf59..17db8f1e2f 100644 --- a/packages/routepolicy/routepolicy_tests.js +++ b/packages/routepolicy/routepolicy_tests.js @@ -2,9 +2,8 @@ Tinytest.add("routepolicy", function (test) { var policy = new Meteor.__RoutePolicyConstructor(); policy.declare('/sockjs/', 'network'); - // App routes might look like this... - // policy.declare('/posts/', 'app'); - // policy.declare('/about', 'app'); + policy.declare('/bigphoto.jpg', 'static-online'); + policy.declare('/anotherphoto.png', 'static-online'); test.equal(policy.classify('/'), null); test.equal(policy.classify('/foo'), null); @@ -13,11 +12,14 @@ Tinytest.add("routepolicy", function (test) { test.equal(policy.classify('/sockjs/'), 'network'); test.equal(policy.classify('/sockjs/foo'), 'network'); - // test.equal(policy.classify('/posts/'), 'app'); - // test.equal(policy.classify('/posts/1234'), 'app'); + test.equal(policy.classify('/bigphoto.jpg'), 'static-online'); + test.equal(policy.classify('/bigphoto.jpg.orig'), 'static-online'); test.equal(policy.urlPrefixesFor('network'), ['/sockjs/']); - // test.equal(policy.urlPrefixesFor('app'), ['/about', '/posts/']); + test.equal( + policy.urlPrefixesFor('static-online'), + ['/anotherphoto.png', '/bigphoto.jpg'] + ); }); Tinytest.add("routepolicy - static conflicts", function (test) { @@ -26,9 +28,14 @@ Tinytest.add("routepolicy - static conflicts", function (test) { "path": "static/sockjs/socks-are-comfy.jpg", "type": "static", "where": "client", - "cacheable": false, "url": "/sockjs/socks-are-comfy.jpg" }, + { + "path": "static/bigphoto.jpg", + "type": "static", + "where": "client", + "url": "/bigphoto.jpg" + } ]; var policy = new Meteor.__RoutePolicyConstructor(); @@ -36,4 +43,9 @@ Tinytest.add("routepolicy - static conflicts", function (test) { policy.checkForConflictWithStatic('/sockjs/', 'network', manifest), "static resource /sockjs/socks-are-comfy.jpg conflicts with network route /sockjs/" ); + + test.equal( + policy.checkForConflictWithStatic('/bigphoto.jpg', 'static-online', manifest), + null + ); });