diff --git a/src/structs/AbstractStruct.js b/src/structs/AbstractStruct.js index c0889f8e..9324a1b4 100644 --- a/src/structs/AbstractStruct.js +++ b/src/structs/AbstractStruct.js @@ -51,6 +51,7 @@ export class AbstractStruct { /** * @param {number} diff + * @return {import('../internals.js').GC|import('../internals.js').Item} */ splice (diff) { throw error.methodUnimplemented() diff --git a/src/structs/ContentAny.js b/src/structs/ContentAny.js index ba7a1b77..4e32ac83 100644 --- a/src/structs/ContentAny.js +++ b/src/structs/ContentAny.js @@ -82,11 +82,12 @@ export class ContentAny { /** * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder * @param {number} offset + * @param {number} offsetEnd */ - write (encoder, offset) { - const len = this.arr.length - encoder.writeLen(len - offset) - for (let i = offset; i < len; i++) { + write (encoder, offset, offsetEnd) { + const end = this.arr.length - offsetEnd + encoder.writeLen(end - offset) + for (let i = offset; i < end; i++) { const c = this.arr[i] encoder.writeAny(c) } diff --git a/src/structs/ContentBinary.js b/src/structs/ContentBinary.js index a376dbbc..580b60fe 100644 --- a/src/structs/ContentBinary.js +++ b/src/structs/ContentBinary.js @@ -71,9 +71,10 @@ export class ContentBinary { gc (_tr) {} /** * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder - * @param {number} offset + * @param {number} _offset + * @param {number} _offsetEnd */ - write (encoder, offset) { + write (encoder, _offset, _offsetEnd) { encoder.writeBuf(this.content) } diff --git a/src/structs/ContentDeleted.js b/src/structs/ContentDeleted.js index 102bcdfa..57b5504b 100644 --- a/src/structs/ContentDeleted.js +++ b/src/structs/ContentDeleted.js @@ -78,9 +78,10 @@ export class ContentDeleted { /** * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder * @param {number} offset + * @param {number} offsetEnd */ - write (encoder, offset) { - encoder.writeLen(this.len - offset) + write (encoder, offset, offsetEnd) { + encoder.writeLen(this.len - offset - offsetEnd) } /** diff --git a/src/structs/ContentDoc.js b/src/structs/ContentDoc.js index 8491272b..6df70556 100644 --- a/src/structs/ContentDoc.js +++ b/src/structs/ContentDoc.js @@ -116,9 +116,10 @@ export class ContentDoc { /** * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder - * @param {number} offset + * @param {number} _offset + * @param {number} _offsetEnd */ - write (encoder, offset) { + write (encoder, _offset, _offsetEnd) { encoder.writeString(this.doc.guid) encoder.writeAny(this.opts) } diff --git a/src/structs/ContentEmbed.js b/src/structs/ContentEmbed.js index 4cb2f922..6d6ce53a 100644 --- a/src/structs/ContentEmbed.js +++ b/src/structs/ContentEmbed.js @@ -74,9 +74,10 @@ export class ContentEmbed { gc (_tr) {} /** * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder - * @param {number} offset + * @param {number} _offset + * @param {number} _offsetEnd */ - write (encoder, offset) { + write (encoder, _offset, _offsetEnd) { encoder.writeJSON(this.embed) } diff --git a/src/structs/ContentFormat.js b/src/structs/ContentFormat.js index d79aa885..1046d255 100644 --- a/src/structs/ContentFormat.js +++ b/src/structs/ContentFormat.js @@ -83,8 +83,9 @@ export class ContentFormat { /** * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder * @param {number} _offset + * @param {number} _offsetEnd */ - write (encoder, _offset) { + write (encoder, _offset, _offsetEnd) { encoder.writeKey(this.key) encoder.writeJSON(this.value) } diff --git a/src/structs/ContentJSON.js b/src/structs/ContentJSON.js index 17591756..f20d7306 100644 --- a/src/structs/ContentJSON.js +++ b/src/structs/ContentJSON.js @@ -79,11 +79,12 @@ export class ContentJSON { /** * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder * @param {number} offset + * @param {number} offsetEnd */ - write (encoder, offset) { - const len = this.arr.length - encoder.writeLen(len - offset) - for (let i = offset; i < len; i++) { + write (encoder, offset, offsetEnd) { + const end = this.arr.length - offsetEnd + encoder.writeLen(end - offset) + for (let i = offset; i < end; i++) { const c = this.arr[i] encoder.writeString(c === undefined ? 'undefined' : JSON.stringify(c)) } diff --git a/src/structs/ContentString.js b/src/structs/ContentString.js index a2f5f368..6f1064b5 100644 --- a/src/structs/ContentString.js +++ b/src/structs/ContentString.js @@ -90,9 +90,10 @@ export class ContentString { /** * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder * @param {number} offset + * @param {number} offsetEnd */ - write (encoder, offset) { - encoder.writeString(offset === 0 ? this.str : this.str.slice(offset)) + write (encoder, offset, offsetEnd) { + encoder.writeString((offset === 0 && offsetEnd === 0) ? this.str : this.str.slice(offset, this.str.length - offsetEnd)) } /** diff --git a/src/structs/ContentType.js b/src/structs/ContentType.js index 13a09cd0..a69677d8 100644 --- a/src/structs/ContentType.js +++ b/src/structs/ContentType.js @@ -149,8 +149,9 @@ export class ContentType { /** * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder * @param {number} _offset + * @param {number} _offsetEnd */ - write (encoder, _offset) { + write (encoder, _offset, _offsetEnd) { this.type._write(encoder) } diff --git a/src/structs/GC.js b/src/structs/GC.js index 8f1d2f9e..7ee1bc33 100644 --- a/src/structs/GC.js +++ b/src/structs/GC.js @@ -3,6 +3,7 @@ import { addStruct, addStructToIdSet, addToIdSet, + createID, UpdateEncoderV1, UpdateEncoderV2, StructStore, Transaction // eslint-disable-line } from '../internals.js' @@ -47,10 +48,11 @@ export class GC extends AbstractStruct { /** * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder * @param {number} offset + * @param {number} offsetEnd */ - write (encoder, offset) { + write (encoder, offset, offsetEnd) { encoder.writeInfo(structGCRefNumber) - encoder.writeLen(this.length - offset) + encoder.writeLen(this.length - offset - offsetEnd) } /** @@ -68,9 +70,11 @@ export class GC extends AbstractStruct { * If this feature is required in the future, then need to try to merge this struct after * transaction. * - * @param {number} _diff + * @param {number} diff */ - splice (_diff) { - return this + splice (diff) { + const other = new GC(createID(this.id.client, this.id.clock + diff), this.length - diff) + this.length = diff + return other } } diff --git a/src/structs/Item.js b/src/structs/Item.js index 042c971c..0efc8e25 100644 --- a/src/structs/Item.js +++ b/src/structs/Item.js @@ -22,7 +22,6 @@ import { readContentType, addChangedTypeToTransaction, addStructToIdSet, - Skip, IdSet, StackItem, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, ContentType, ContentDeleted, StructStore, ID, AbstractType, Transaction, // eslint-disable-line } from '../internals.js' @@ -126,6 +125,26 @@ export const splitItem = (transaction, leftItem, diff) => { return rightItem } +/** + * More generalized version of splitItem. Split leftStruct into two structs + * @param {Transaction?} transaction + * @param {AbstractStruct} leftStruct + * @param {number} diff + * @return {GC|Item} + * + * @function + * @private + */ +export const splitStruct = (transaction, leftStruct, diff) => { + if (leftStruct instanceof Item) { + return splitItem(transaction, leftStruct, diff) + } else { + const rightItem = leftStruct.splice(diff) + transaction?._mergeStructs.push(rightItem) + return rightItem + } +} + /** * @param {Array} stack * @param {ID} id @@ -409,11 +428,6 @@ export class Item extends AbstractStruct { this.parent = /** @type {ContentType} */ (parentItem.content).type } } - // @todo remove thgis - if (this.left instanceof Skip || this.right instanceof Skip || this.parent instanceof Skip) { - debugger - throw new Error('dtruinae') - } return null } @@ -634,7 +648,7 @@ export class Item extends AbstractStruct { } /** - * @param {Transaction} tr + * @param {Transaction} tr * @param {boolean} parentGCd */ gc (tr, parentGCd) { @@ -657,8 +671,9 @@ export class Item extends AbstractStruct { * * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder The encoder to write data to. * @param {number} offset + * @param {number} offsetEnd */ - write (encoder, offset) { + write (encoder, offset, offsetEnd) { const origin = offset > 0 ? createID(this.id.client, this.id.clock + offset - 1) : this.origin const rightOrigin = this.rightOrigin const parentSub = this.parentSub @@ -700,7 +715,7 @@ export class Item extends AbstractStruct { encoder.writeString(parentSub) } } - this.content.write(encoder, offset) + this.content.write(encoder, offset, offsetEnd) } } @@ -808,8 +823,9 @@ export class AbstractContent { /** * @param {UpdateEncoderV1 | UpdateEncoderV2} _encoder * @param {number} _offset + * @param {number} _offsetEnd */ - write (_encoder, _offset) { + write (_encoder, _offset, _offsetEnd) { throw error.methodUnimplemented() } diff --git a/src/utils/Doc.js b/src/utils/Doc.js index 3381e250..c116f3f9 100644 --- a/src/utils/Doc.js +++ b/src/utils/Doc.js @@ -62,7 +62,7 @@ export class Doc extends ObservableV2 { /** * @param {DocOpts} opts configuration */ - constructor ({ guid = random.uuidv4(), collectionid = null, gc = true, gcFilter = () => true, meta = null, autoLoad = false, shouldLoad = true, isSuggestionDoc = false} = {}) { + constructor ({ guid = random.uuidv4(), collectionid = null, gc = true, gcFilter = () => true, meta = null, autoLoad = false, shouldLoad = true, isSuggestionDoc = false } = {}) { super() this.gc = gc this.gcFilter = gcFilter diff --git a/src/utils/Snapshot.js b/src/utils/Snapshot.js index fcf9ea0a..c585a9a2 100644 --- a/src/utils/Snapshot.js +++ b/src/utils/Snapshot.js @@ -184,7 +184,7 @@ export const createDocFromSnapshot = (originDoc, snapshot, newDoc = new Doc()) = // first clock written is 0 encoding.writeVarUint(encoder.restEncoder, 0) for (let i = 0; i <= lastStructIndex; i++) { - structs[i].write(encoder, 0) + structs[i].write(encoder, 0, 0) } } writeIdSet(encoder, ds) diff --git a/src/utils/StructSet.js b/src/utils/StructSet.js index 5f602810..4af31748 100644 --- a/src/utils/StructSet.js +++ b/src/utils/StructSet.js @@ -83,6 +83,7 @@ export const readStructSet = (decoder, doc) => { * @param {IdSet} exclude */ export const removeRangesFromStructSet = (ss, exclude) => { + // @todo walk through ss instead to reduce iterations exclude.clients.forEach((range, client) => { const structs = /** @type {StructRange} */ (ss.clients.get(client))?.refs if (structs != null) { diff --git a/src/utils/StructStore.js b/src/utils/StructStore.js index fdcd7a5a..78b734a7 100644 --- a/src/utils/StructStore.js +++ b/src/utils/StructStore.js @@ -5,7 +5,8 @@ import { createIdSet, Transaction, ID, Item, // eslint-disable-line Skip, - createID + createID, + splitStruct } from '../internals.js' import * as math from 'lib0/math' @@ -198,8 +199,8 @@ export const getItem = /** @type {function(StructStore,ID):Item} */ (find) export const findIndexCleanStart = (transaction, structs, clock) => { const index = findIndexSS(structs, clock) const struct = structs[index] - if (struct.id.clock < clock && struct.constructor !== GC) { - structs.splice(index + 1, 0, struct instanceof Item ? splitItem(transaction, struct, clock - struct.id.clock) : struct.splice(clock - struct.id.clock)) + if (struct.id.clock < clock) { + structs.splice(index + 1, 0, splitStruct(transaction, struct, clock - struct.id.clock)) return index + 1 } return index diff --git a/src/utils/Transaction.js b/src/utils/Transaction.js index 98caf979..f6b59095 100644 --- a/src/utils/Transaction.js +++ b/src/utils/Transaction.js @@ -10,8 +10,7 @@ import { generateNewClientId, createID, cleanupYTextAfterTransaction, - IdSet, UpdateEncoderV1, UpdateEncoderV2, GC, StructStore, AbstractType, AbstractStruct, YEvent, Doc - // insertIntoIdSet + IdSet, UpdateEncoderV1, UpdateEncoderV2, GC, StructStore, AbstractType, AbstractStruct, YEvent, Doc // eslint-disable-line } from '../internals.js' import * as error from 'lib0/error' diff --git a/src/utils/encoding.js b/src/utils/encoding.js index 163cafab..285e8259 100644 --- a/src/utils/encoding.js +++ b/src/utils/encoding.js @@ -37,7 +37,8 @@ import { removeRangesFromStructSet, createIdSet, StructSet, IdSet, DSDecoderV2, Doc, Transaction, GC, Item, StructStore, // eslint-disable-line - createID + createID, + IdRange } from '../internals.js' import * as encoding from 'lib0/encoding' @@ -50,24 +51,57 @@ import * as array from 'lib0/array' * @param {UpdateEncoderV1 | UpdateEncoderV2} encoder * @param {Array} structs All structs by `client` * @param {number} client - * @param {number} clock write structs starting with `ID(client,clock)` + * @param {Array} idranges * * @function */ -const writeStructs = (encoder, structs, client, clock) => { - // write first id - clock = math.max(clock, structs[0].id.clock) // make sure the first id exists - const startNewStructs = findIndexSS(structs, clock) +const writeStructs = (encoder, structs, client, idranges) => { + let structsToWrite = 0 // this accounts for the skips + /** + * @type {Array<{ start: number, end: number, startClock: number, endClock: number }>} + */ + const indexRanges = [] + const firstPossibleClock = structs[0].id.clock + const lastStruct = array.last(structs) + const lastPossibleClock = lastStruct.id.clock + lastStruct.length + idranges.forEach(idrange => { + const startClock = math.max(idrange.clock, firstPossibleClock) + const endClock = math.min(idrange.clock + idrange.len, lastPossibleClock) + if (startClock >= endClock) return // structs for this range do not exist + // inclusive start + const start = findIndexSS(structs, startClock) + // exclusive end + const end = findIndexSS(structs, endClock - 1) + 1 + structsToWrite += end - start + indexRanges.push({ + start, + end, + startClock, + endClock + }) + }) + structsToWrite += idranges.length - 1 + // start writing with this clock. this is updated to the next clock that we expect to write + let clock = indexRanges[0].startClock // write # encoded structs - encoding.writeVarUint(encoder.restEncoder, structs.length - startNewStructs) + encoding.writeVarUint(encoder.restEncoder, structsToWrite) encoder.writeClient(client) + // write clock encoding.writeVarUint(encoder.restEncoder, clock) - const firstStruct = structs[startNewStructs] - // write first struct with an offset - firstStruct.write(encoder, clock - firstStruct.id.clock) - for (let i = startNewStructs + 1; i < structs.length; i++) { - structs[i].write(encoder, 0) - } + indexRanges.forEach(indexRange => { + const skipLen = indexRange.startClock - clock + if (skipLen > 0) { + new Skip(createID(client, clock), skipLen).write(encoder, 0) + clock += skipLen + } + for (let i = indexRange.start; i < indexRange.end; i++) { + const struct = structs[i] + const structEnd = struct.id.clock + struct.length + const offsetEnd = math.max(structEnd - indexRange.endClock, 0) + struct.write(encoder, clock - struct.id.clock, offsetEnd) + clock = structEnd - offsetEnd + } + }) } /** @@ -97,7 +131,9 @@ export const writeClientsStructs = (encoder, store, _sm) => { // Write items with higher client ids first // This heavily improves the conflict algorithm. array.from(sm.entries()).sort((a, b) => b[0] - a[0]).forEach(([client, clock]) => { - writeStructs(encoder, /** @type {Array} */ (store.clients.get(client)), client, clock) + const structs = /** @type {Array} */ (store.clients.get(client)) + const lastStruct = structs[structs.length - 1] + writeStructs(encoder, structs, client, [new IdRange(clock, lastStruct.id.clock + lastStruct.length - clock)]) }) } @@ -117,7 +153,9 @@ export const writeStructsFromIdSet = (encoder, store, idset) => { // Write items with higher client ids first // This heavily improves the conflict algorithm. array.from(idset.clients.entries()).sort((a, b) => b[0] - a[0]).forEach(([client, ids]) => { - writeStructs(encoder, /** @type {Array} */ (store.clients.get(client)), client, ids.getIds()[0].clock) + const idRanges = ids.getIds() + const structs = /** @type {Array} */ (store.clients.get(client)) + writeStructs(encoder, structs, client, idRanges) }) } diff --git a/src/utils/updates.js b/src/utils/updates.js index 558c9c2c..ba6ef59d 100644 --- a/src/utils/updates.js +++ b/src/utils/updates.js @@ -534,7 +534,7 @@ const writeStructToLazyStructWriter = (lazyWriter, struct, offset) => { // write startClock encoding.writeVarUint(lazyWriter.encoder.restEncoder, struct.id.clock + offset) } - struct.write(lazyWriter.encoder, offset) + struct.write(lazyWriter.encoder, offset, 0) lazyWriter.written++ } /** diff --git a/tests/updates.tests.js b/tests/updates.tests.js index 40ccc58a..f68c1978 100644 --- a/tests/updates.tests.js +++ b/tests/updates.tests.js @@ -180,7 +180,6 @@ const checkUpdateCases = (ydoc, updates, enc, hasDeletes) => { for (let j = 1; j < updates.length; j++) { const partMerged = enc.mergeUpdates(updates.slice(j)) const partMeta = enc.parseUpdateMeta(partMerged) - const targetSV = enc.encodeStateVectorFromUpdate(enc.mergeUpdates(updates.slice(0, j))) const diffed = enc.diffUpdate(mergedUpdates, targetSV) const diffedMeta = enc.parseUpdateMeta(diffed)