diff --git a/src/utils/IdMap.js b/src/utils/IdMap.js index d91f9e02..8459a6ee 100644 --- a/src/utils/IdMap.js +++ b/src/utils/IdMap.js @@ -1,12 +1,13 @@ import { _diffSet, findIndexInIdRanges, - IdSet, ID // eslint-disable-line + DSDecoderV1, DSDecoderV2, DSEncoderV1, DSEncoderV2, IdSet, ID // eslint-disable-line } from '../internals.js' import * as array from 'lib0/array' import * as map from 'lib0/map' import * as encoding from 'lib0/encoding' +import * as decoding from 'lib0/decoding' import * as buf from 'lib0/buffer' import * as rabin from 'lib0/hash/rabin' @@ -41,7 +42,6 @@ const _hashAttribution = attr => { return buf.toBase64(rabin.fingerprint(rabin.StandardIrreducible128, encoding.toUint8Array(encoder))) } - /** * @template V * @param {string} name @@ -126,6 +126,7 @@ export class AttrRanges { * @param {Array>} attrs */ add (clock, length, attrs) { + if (length === 0) return this.sorted = false this._ids.push(new AttrRange(clock, length, attrs)) } @@ -325,6 +326,7 @@ export class IdMap { * @param {Array>} attrs */ add (client, clock, len, attrs) { + if (len === 0) return attrs = _ensureAttrs(this, attrs) const ranges = this.clients.get(client) if (ranges == null) { @@ -335,6 +337,146 @@ export class IdMap { } } +/** + * Efficiently encodes IdMap to a binary form. Ensures that information is de-duplicated when + * written. Attribute.names are referenced by id. Attributes themselfs are also referenced by id. + * + * @template Attr + * @param {DSEncoderV1 | DSEncoderV2} encoder + * @param {IdMap} idmap + * + * @private + * @function + */ +export const writeIdMap = (encoder, idmap) => { + encoding.writeVarUint(encoder.restEncoder, idmap.clients.size) + let lastWrittenClientId = 0 + /** + * @type {Map, number>} + */ + const visitedAttributions = map.create() + /** + * @type {Map} + */ + const visitedAttrNames = map.create() + // Ensure that the delete set is written in a deterministic order (smaller clientids first) + array.from(idmap.clients.entries()) + .sort((a, b) => a[0] - b[0]) + .forEach(([client, _idRanges]) => { + const attrRanges = _idRanges.getIds() + encoder.resetDsCurVal() + const diff = client - lastWrittenClientId + encoding.writeVarUint(encoder.restEncoder, diff) + lastWrittenClientId = client + const len = attrRanges.length + encoding.writeVarUint(encoder.restEncoder, len) + for (let i = 0; i < len; i++) { + const item = attrRanges[i] + const attrs = item.attrs + const attrLen = attrs.length + encoder.writeDsClock(item.clock) + encoder.writeDsLen(item.len) + encoding.writeVarUint(encoder.restEncoder, attrLen) + for (let j = 0; j < attrLen; j++) { + const attr = attrs[j] + const attrId = visitedAttributions.get(attr) + if (attrId != null) { + encoding.writeVarUint(encoder.restEncoder, attrId) + } else { + const newAttrId = visitedAttributions.size + visitedAttributions.set(attr, newAttrId) + encoding.writeVarUint(encoder.restEncoder, newAttrId) + const attrNameId = visitedAttrNames.get(attr.name) + // write attr.name + if (attrNameId != null) { + encoding.writeVarUint(encoder.restEncoder, attrNameId) + } else { + const newAttrNameId = visitedAttrNames.size + encoding.writeVarUint(encoder.restEncoder, newAttrNameId) + encoding.writeVarString(encoder.restEncoder, attr.name) + visitedAttrNames.set(attr.name, newAttrNameId) + } + encoding.writeAny(encoder.restEncoder, /** @type {any} */ (attr.val)) + } + } + } + }) +} + +/** + * @param {IdMap} idmap + */ +export const encodeIdMap = idmap => { + const encoder = new DSEncoderV2() + writeIdMap(encoder, idmap) + return encoder.toUint8Array() +} + +/** + * @param {DSDecoderV1 | DSDecoderV2} decoder + * @return {IdMap} + * + * @private + * @function + */ +export const readIdMap = decoder => { + const idmap = new IdMap() + const numClients = decoding.readVarUint(decoder.restDecoder) + /** + * @type {Array>} + */ + const visitedAttributions = [] + /** + * @type {Array} + */ + const visitedAttrNames = [] + let lastClientId = 0 + for (let i = 0; i < numClients; i++) { + decoder.resetDsCurVal() + const client = lastClientId + decoding.readVarUint(decoder.restDecoder) + lastClientId = client + const numberOfDeletes = decoding.readVarUint(decoder.restDecoder) + /** + * @type {Array>} + */ + const attrRanges = [] + for (let i = 0; i < numberOfDeletes; i++) { + const rangeClock = decoder.readDsClock() + const rangeLen = decoder.readDsLen() + /** + * @type {Array>} + */ + const attrs = [] + const attrsLen = decoding.readVarUint(decoder.restDecoder) + for (let j = 0; j < attrsLen; j++) { + const attrId = decoding.readVarUint(decoder.restDecoder) + if (attrId >= visitedAttributions.length) { + // attrId not known yet + const attrNameId = decoding.readVarUint(decoder.restDecoder) + if (attrNameId >= visitedAttrNames.length) { + visitedAttrNames.push(decoding.readVarString(decoder.restDecoder)) + } + visitedAttributions.push(new Attribution(visitedAttrNames[attrNameId], decoding.readAny(decoder.restDecoder))) + } + attrs.push(visitedAttributions[attrId]) + } + attrRanges.push(new AttrRange(rangeClock, rangeLen, attrs)) + } + idmap.clients.set(client, new AttrRanges(attrRanges)) + } + visitedAttributions.forEach(attr => { + idmap.attrs.add(attr) + idmap.attrsH.set(attr.hash(), attr) + }) + return idmap +} + +/** + * @param {Uint8Array} data + * @return {IdMap} + */ +export const decodeIdMap = data => readIdMap(new DSDecoderV2(decoding.createDecoder(data))) + /** * @template Attrs * @param {IdMap} idmap @@ -342,10 +484,12 @@ export class IdMap { * @return {Array>} */ const _ensureAttrs = (idmap, attrs) => attrs.map(attr => - idmap.attrs.has(attr) ? attr : map.setIfUndefined(idmap.attrsH, _hashAttribution(attr), () => { - idmap.attrs.add(attr) - return attr - })) + idmap.attrs.has(attr) + ? attr + : map.setIfUndefined(idmap.attrsH, _hashAttribution(attr), () => { + idmap.attrs.add(attr) + return attr + })) export const createIdMap = () => new IdMap() @@ -358,4 +502,9 @@ export const createIdMap = () => new IdMap() * @param {IdSet | IdMap} exclude * @return {ISet} */ -export const diffIdMap = _diffSet +export const diffIdMap = (set, exclude) => { + const diffed = _diffSet(set, exclude) + diffed.attrs = set.attrs + diffed.attrsH = set.attrsH + return diffed +} diff --git a/src/utils/IdSet.js b/src/utils/IdSet.js index 21efe141..b17a0f6a 100644 --- a/src/utils/IdSet.js +++ b/src/utils/IdSet.js @@ -303,6 +303,7 @@ export const diffIdSet = _diffSet * @function */ export const addToIdSet = (idSet, client, clock, length) => { + if (length === 0) return const idRanges = idSet.clients.get(client) if (idRanges) { idRanges.add(clock, length) diff --git a/tests/IdMap.tests.js b/tests/IdMap.tests.js index 24388e32..b8c7578d 100644 --- a/tests/IdMap.tests.js +++ b/tests/IdMap.tests.js @@ -1,6 +1,7 @@ import * as t from 'lib0/testing' import * as am from '../src/utils/IdMap.js' import { compareIdmaps, createIdMap, ID, createRandomIdSet, createRandomIdMap, createAttribution } from './testHelper.js' +import * as YY from '../src/internals.js' /** * @template T @@ -115,6 +116,8 @@ export const testRepeatRandomDiffing = tc => { const e1 = am.diffIdMap(ds1, ds2) const e2 = am.diffIdMap(merged, ds2) compareIdmaps(e1, e2) + const copy = YY.decodeIdMap(YY.encodeIdMap(e1)) + compareIdmaps(e1, copy) } /** @@ -133,4 +136,6 @@ export const testRepeatRandomDiffing2 = tc => { const e2 = am.diffIdMap(am2, idsExclude) const excludedMerged = am.mergeIdMaps([e1, e2]) compareIdmaps(mergedExcluded, excludedMerged) + const copy = YY.decodeIdMap(YY.encodeIdMap(mergedExcluded)) + compareIdmaps(mergedExcluded, copy) } diff --git a/tests/testHelper.js b/tests/testHelper.js index 6e55c746..9be857ea 100644 --- a/tests/testHelper.js +++ b/tests/testHelper.js @@ -8,7 +8,7 @@ import * as map from 'lib0/map' import * as Y from '../src/index.js' import * as math from 'lib0/math' import { - createIdSet, createIdMap, addToIdSet + createIdSet, createIdMap, addToIdSet, encodeIdMap } from '../src/internals.js' export * from '../src/index.js' @@ -313,7 +313,7 @@ export const init = (tc, { users = 5 } = {}, initTestObject) => { * @param {Y.IdSet} idSet2 */ export const compareIdSets = (idSet1, idSet2) => { - if (idSet1.clients.size !== idSet2.clients.size) return false + t.assert(idSet1.clients.size === idSet2.clients.size) for (const [client, _items1] of idSet1.clients.entries()) { const items1 = _items1.getIds() const items2 = idSet2.clients.get(client)?.getIds() @@ -349,13 +349,34 @@ const _idmapAttrsHas = (attrs, attr) => { */ export const _idmapAttrsEqual = (a, b) => a.length === b.length && a.every(v => _idmapAttrsHas(b, v)) +/** + * Ensure that all attributes exist. Also create a copy and compare it to the original. + * + * @template T + * @param {Y.IdMap} idmap + */ +export const validateIdMap = idmap => { + const copy = Y.createIdMap() + idmap.clients.forEach((ranges, client) => { + ranges.getIds().forEach(range => { + range.attrs.forEach(attr => { + t.assert(idmap.attrs.has(attr)) + t.assert(idmap.attrsH.get(attr.hash()) === attr) + copy.add(client, range.clock, range.len, range.attrs.slice()) + }) + }) + t.assert(copy.clients.get(client)?.getIds().length === ranges.getIds().length) + }) + t.assert(idmap.attrsH.size === idmap.attrs.size) +} + /** * @template T * @param {Y.IdMap} am1 * @param {Y.IdMap} am2 */ export const compareIdmaps = (am1, am2) => { - if (am1.clients.size !== am2.clients.size) return false + t.assert(am1.clients.size === am2.clients.size) for (const [client, _items1] of am1.clients.entries()) { const items1 = _items1.getIds() const items2 = am2.clients.get(client)?.getIds() @@ -366,7 +387,8 @@ export const compareIdmaps = (am1, am2) => { t.assert(di1.clock === di2.clock && di1.len === di2.len && _idmapAttrsEqual(di1.attrs, di2.attrs)) } } - return true + validateIdMap(am1) + validateIdMap(am2) } /** @@ -416,6 +438,7 @@ export const createRandomIdMap = (gen, clients, clockRange, attrChoices) => { } idMap.add(client, clockStart, len, attrs.map(v => Y.createAttribution('', v))) } + t.info(`Created IdMap with ${numOfOps} ranges and ${attrChoices.length} different attributes. Encoded size: ${encodeIdMap(idMap).byteLength}`) return idMap }