github/sim
1
1
Fork 0
mirror of https://github.com/simstudioai/sim.git synced 2026-04-28 03:00:29 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(polling): prevent data loss on partial row failures and harden idempotency key

Browse Source 
- Sheets: only advance lastKnownRowCount by processedCount when there
  are failures, so failed rows are retried on the next poll cycle
  (idempotency deduplicates already-processed rows on re-fetch)
- Drive: add fallback for change.time in idempotency key to prevent
  key collisions if the field is ever absent from the API response
This commit is contained in:
Waleed Latif
2026-04-09 14:43:21 -07:00
committed by waleed
parent cfcc208728
commit 9df3c078ae
2 changed files with 11 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/sim/lib/webhooks/polling/google-drive.ts
View File

@@ -357,7 +357,7 @@ async function processChanges(
}
try {
const idempotencyKey = `${webhookData.id}:${change.fileId}:${change.time}`
const idempotencyKey = `${webhookData.id}:${change.fileId}:${change.time || change.fileId}`
await pollingIdempotency.executeWithIdempotency('google-drive', idempotencyKey, async () => {
const payload: GoogleDriveWebhookPayload = {

17
apps/sim/lib/webhooks/polling/google-sheets.ts
View File

@@ -182,17 +182,20 @@ export const googleSheetsPollingHandler: PollingProviderHandler = {
logger
)
// Update state: advance row count by the number we fetched (not total new rows)
// so remaining rows are picked up in the next poll.
// When batching (more rows than maxRowsPerPoll), keep the old lastModifiedTime
// so the Drive pre-check doesn't skip remaining rows on the next poll.
const newLastKnownRowCount = config.lastKnownRowCount + rowsToFetch
const hasRemainingRows = rowsToFetch < newRowCount
// Advance row count only by successfully processed rows so failed rows
// can be retried on the next poll cycle. Idempotency deduplicates the
// already-processed rows when they are re-fetched.
const rowsAdvanced = failedCount > 0 ? processedCount : rowsToFetch
const newLastKnownRowCount = config.lastKnownRowCount + rowsAdvanced
// When batching (more rows than maxRowsPerPoll) or retrying failed rows,
// keep the old lastModifiedTime so the Drive pre-check doesn't skip
// remaining/retried rows on the next poll.
const hasRemainingOrFailed = rowsAdvanced < newRowCount
await updateWebhookProviderConfig(
webhookId,
{
lastKnownRowCount: newLastKnownRowCount,
lastModifiedTime: hasRemainingRows ? config.lastModifiedTime : currentModifiedTime,
lastModifiedTime: hasRemainingOrFailed ? config.lastModifiedTime : currentModifiedTime,
lastCheckedTimestamp: now.toISOString(),
},
logger