fix(backend): split multi-key DELETE to avoid CROSSSLOT under Redis Cluster

Addresses two sentry bug predictions on the latest commit:

1. `reset_user_usage(reset_weekly=True)` deleted the daily and weekly
   keys in one `redis.delete(d_key, w_key)` call. Those two keys hash to
   different cluster slots, so the call fails with CROSSSLOT and the
   admin-facing reset endpoint returns 500.

2. `clear_insufficient_funds_notifications` collects N keys for a user
   (one per graph) via SCAN and bulk-deletes them. Each graph's key
   hashes independently, so the bulk DELETE raises CROSSSLOT whenever
   the user has notifications for more than one graph.

Both now issue per-key DELETE calls. The operations are idempotent and
neither needs cross-key atomicity — a crash mid-loop just leaves a few
stale keys with TTLs that will expire anyway.
This commit is contained in:
majdyz
2026-04-24 02:41:51 +07:00
parent 1fd74b8a82
commit e958b14903
2 changed files with 13 additions and 7 deletions

View File

@@ -721,12 +721,15 @@ async def reset_user_usage(user_id: str, *, reset_weekly: bool = False) -> None:
the admin believing the counters were zeroed when they were not.
"""
now = datetime.now(UTC)
keys_to_delete = [_daily_key(user_id, now=now)]
if reset_weekly:
keys_to_delete.append(_weekly_key(user_id, now=now))
d_key = _daily_key(user_id, now=now)
w_key = _weekly_key(user_id, now=now) if reset_weekly else None
try:
redis = await get_redis_async()
await redis.delete(*keys_to_delete)
# Daily and weekly keys hash to different cluster slots — multi-key
# DELETE would raise CROSSSLOT, so issue separate calls.
await redis.delete(d_key)
if w_key is not None:
await redis.delete(w_key)
except (RedisError, ConnectionError, OSError):
logger.warning("Redis unavailable for resetting user usage")
raise

View File

@@ -74,9 +74,12 @@ async def clear_insufficient_funds_notifications(user_id: str) -> int:
redis_client = await redis.get_redis_async()
pattern = f"{INSUFFICIENT_FUNDS_NOTIFIED_PREFIX}:{user_id}:*"
keys = [key async for key in redis_client.scan_iter(match=pattern)]
if keys:
return await redis_client.delete(*keys)
return 0
# Keys here span multiple graph IDs and therefore multiple cluster
# slots — a bulk DELETE would raise CROSSSLOT, so delete per key.
deleted = 0
for key in keys:
deleted += await redis_client.delete(key)
return deleted
except Exception as e:
logger.warning(
f"Failed to clear insufficient funds notification flags for user "