mirror of
https://github.com/meteor/meteor.git
synced 2026-05-02 03:01:46 -04:00
30 lines
827 B
JavaScript
30 lines
827 B
JavaScript
// @export HTML5Tokenizer
|
|
HTML5Tokenizer = {
|
|
tokenize: function (inputString) {
|
|
var tokens = [];
|
|
var tokenizer = new HTML5.Tokenizer(inputString);
|
|
tokenizer.addListener('token', function (tok) {
|
|
tokens.push(tok);
|
|
});
|
|
tokenizer.tokenize();
|
|
return tokens;
|
|
}
|
|
// Incremental tokenization turns out not to be useful
|
|
// for inspecting intermediate tokenizer state, just
|
|
// for async streaming.
|
|
//
|
|
// tokenizeIncremental: function (tokenFunc) {
|
|
// var emitter = new toyevents.EventEmitter();
|
|
// var tokenizer = new HTML5.Tokenizer(emitter);
|
|
// tokenizer.addListener('token', tokenFunc);
|
|
// return {
|
|
// add: function (str) {
|
|
// emitter.emit('data', str);
|
|
// },
|
|
// finish: function () {
|
|
// emitter.emit('end');
|
|
// }
|
|
// };
|
|
// }
|
|
};
|