diff --git a/.app_version b/.app_version index e780e0e0..26bea73e 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.30.12 +0.31.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index ad662742..37fdb788 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -# [UNRELEASED] +# [0.31.0] - 2025-09-04 The Search release @@ -20,6 +20,11 @@ Important: This feature relies on reverse geocoding. Without reverse geocoding, - Default value for `points_count` attribute is now set to 0 in the User model. +## Changed + +- Tracks are not being calculated by server instead of the database. This feature is still in progress. + + # [0.30.12] - 2025-08-26 ## Fixed diff --git a/PARALLEL_TRACK_GENERATOR_PLAN.md b/PARALLEL_TRACK_GENERATOR_PLAN.md deleted file mode 100644 index 9fd5bfd3..00000000 --- a/PARALLEL_TRACK_GENERATOR_PLAN.md +++ /dev/null @@ -1,387 +0,0 @@ -# Parallel Track Generator Implementation Plan - -## ✅ IMPLEMENTATION COMPLETED - -This document outlines the implementation plan for building an alternative track generator that moves heavy database operations to Ruby-side processing with background job support. The new system will process tracks in parallel time-based chunks while maintaining track integrity across boundaries. - -**Status: ✅ COMPLETE** - All core functionality has been implemented and tested successfully. - -## Current State Analysis - -### Existing Implementation Issues -- Heavy reliance on complex SQL operations in `Track.get_segments_with_points` (app/services/tracks/generator.rb:47) -- Uses PostgreSQL window functions, geography calculations, and array aggregations -- All processing happens in a single synchronous operation -- Memory intensive for large datasets -- No parallel processing capability - -### Dependencies Available -- ✅ ActiveJob framework already in use -- ✅ Geocoder gem available for distance calculations -- ✅ Existing job patterns (see app/jobs/tracks/create_job.rb) -- ✅ User settings for time/distance thresholds - -## Architecture Overview - -### ✅ Implemented Directory Structure -``` -app/ -├── jobs/ -│ └── tracks/ -│ ├── parallel_generator_job.rb ✅ Main coordinator -│ ├── time_chunk_processor_job.rb ✅ Process individual time chunks -│ └── boundary_resolver_job.rb ✅ Merge cross-chunk tracks -├── services/ -│ └── tracks/ -│ ├── parallel_generator.rb ✅ Main service class -│ ├── time_chunker.rb ✅ Split time ranges into chunks -│ ├── segmentation.rb ✅ Ruby-based point segmentation (extended existing) -│ ├── boundary_detector.rb ✅ Handle cross-chunk boundaries -│ ├── session_manager.rb ✅ Rails.cache-based session tracking -│ └── session_cleanup.rb ✅ Background maintenance tasks -└── models/concerns/ - └── distanceable.rb ✅ Extended with Geocoder calculations -``` - -### ✅ Implemented Key Components - -1. **✅ Parallel Generator**: Main orchestrator service - coordinates the entire parallel process -2. **✅ Time Chunker**: Splits date ranges into processable chunks with buffer zones (default: 1 day) -3. **✅ Rails.cache Session Manager**: Tracks job progress and coordination (instead of Redis) -4. **✅ Enhanced Segmentation**: Extended existing module with Geocoder-based calculations -5. **✅ Chunk Processor Jobs**: Process individual time chunks in parallel using ActiveJob -6. **✅ Boundary Resolver**: Handles tracks spanning multiple chunks with sophisticated merging logic -7. **✅ Session Cleanup**: Background maintenance and health monitoring - -### ✅ Implemented Data Flow -``` -User Request - ↓ -ParallelGeneratorJob ✅ - ↓ -Creates Rails.cache session entry ✅ - ↓ -TimeChunker splits date range with buffer zones ✅ - ↓ -Multiple TimeChunkProcessorJob (parallel) ✅ - ↓ -Each processes one time chunk using Geocoder ✅ - ↓ -BoundaryResolverJob (waits for all chunks) ✅ - ↓ -Merges cross-boundary tracks ✅ - ↓ -Rails.cache session marked as completed ✅ -``` - -## Implementation Plan - -### Phase 1: Foundation (High Priority) - -#### 1.1 Redis-Based Session Tracking -**Files to create:** -- `app/services/tracks/session_manager.rb` - -**Redis Schema:** -```ruby -# Key pattern: "track_generation:user:#{user_id}:#{session_id}" -{ - status: "pending", # pending, processing, completed, failed - total_chunks: 0, - completed_chunks: 0, - tracks_created: 0, - started_at: "2024-01-01T10:00:00Z", - completed_at: nil, - error_message: nil, - metadata: { - mode: "bulk", - chunk_size: "1.day", - user_settings: {...} - } -} - -#### 1.2 Extend Distanceable Concern -**File:** `app/models/concerns/distanceable.rb` -- Add Geocoder-based Ruby calculation methods -- Support pure Ruby distance calculations without SQL -- Maintain compatibility with existing PostGIS methods - -#### 1.3 Time Chunker Service -**File:** `app/services/tracks/time_chunker.rb` -- Split time ranges into configurable chunks (default: 1 day) -- Add buffer zones for boundary detection (6-hour overlap) -- Handle edge cases (empty ranges, single day) - -### Phase 2: Core Processing (High Priority) - -#### 2.1 Ruby Segmentation Service -**File:** `app/services/tracks/ruby_segmentation.rb` -- Replace SQL window functions with Ruby logic -- Stream points using `find_each` for memory efficiency -- Use Geocoder for distance calculations -- Implement gap detection (time and distance thresholds) -- Return segments with pre-calculated distances - -#### 2.2 Parallel Generator Service -**File:** `app/services/tracks/parallel_generator.rb` -- Main orchestrator for the entire process -- Create generation sessions -- Coordinate job enqueueing -- Support all existing modes (bulk, incremental, daily) - -### Phase 3: Background Jobs (High Priority) - -#### 3.1 Parallel Generator Job -**File:** `app/jobs/tracks/parallel_generator_job.rb` -- Entry point for background processing -- Replace existing `Tracks::CreateJob` usage -- Handle user notifications - -#### 3.2 Time Chunk Processor Job -**File:** `app/jobs/tracks/time_chunk_processor_job.rb` -- Process individual time chunks -- Create tracks from segments -- Update session progress -- Handle chunk-level errors - -#### 3.3 Boundary Resolver Job -**File:** `app/jobs/tracks/boundary_resolver_job.rb` -- Wait for all chunks to complete -- Identify and merge cross-boundary tracks -- Clean up duplicate/overlapping tracks -- Finalize session - -### Phase 4: Enhanced Features (Medium Priority) - -#### 4.1 Boundary Detector Service -**File:** `app/services/tracks/boundary_detector.rb` -- Detect tracks spanning multiple chunks -- Merge partial tracks across boundaries -- Avoid duplicate track creation -- Handle complex multi-day journeys - -#### 4.2 Session Cleanup Service -**File:** `app/services/tracks/session_cleanup.rb` -- Handle stuck/failed sessions -- Cleanup expired Redis sessions -- Background maintenance tasks - -### Phase 5: Integration & Testing (Medium Priority) - -#### 5.1 Controller Integration -- Update existing controllers to use parallel generator -- Maintain backward compatibility -- Simple status checking if needed - -#### 5.2 Error Handling & Retry Logic -- Implement exponential backoff for failed chunks -- Add dead letter queue for permanent failures -- Create rollback mechanisms -- Comprehensive logging and monitoring - -#### 5.3 Performance Optimization -- Benchmark memory usage vs SQL approach -- Test scalability with large datasets -- Profile job queue performance -- Optimize Geocoder usage - -## ✅ IMPLEMENTATION STATUS - -### Foundation Tasks ✅ COMPLETE -- [x] **✅ DONE** Create `Tracks::SessionManager` service for Rails.cache-based tracking -- [x] **✅ DONE** Implement session creation, updates, and cleanup -- [x] **✅ DONE** Extend `Distanceable` concern with Geocoder integration -- [x] **✅ DONE** Implement `Tracks::TimeChunker` with buffer zones -- [x] **✅ DONE** Add Rails.cache TTL and cleanup strategies -- [x] **✅ DONE** Write comprehensive unit tests (34/34 SessionManager, 20/20 TimeChunker tests passing) - -### Core Processing Tasks ✅ COMPLETE -- [x] **✅ DONE** Extend `Tracks::Segmentation` with Geocoder-based methods -- [x] **✅ DONE** Replace SQL operations with Ruby streaming logic -- [x] **✅ DONE** Add point loading with batching support -- [x] **✅ DONE** Implement gap detection using time/distance thresholds -- [x] **✅ DONE** Create `Tracks::ParallelGenerator` orchestrator service -- [x] **✅ DONE** Support all existing modes (bulk, incremental, daily) -- [x] **✅ DONE** Write comprehensive unit tests (37/37 ParallelGenerator tests passing) - -### Background Job Tasks ✅ COMPLETE -- [x] **✅ DONE** Create `Tracks::ParallelGeneratorJob` entry point -- [x] **✅ DONE** Implement `Tracks::TimeChunkProcessorJob` for parallel processing -- [x] **✅ DONE** Add progress tracking and error handling -- [x] **✅ DONE** Create `Tracks::BoundaryResolverJob` for cross-chunk merging -- [x] **✅ DONE** Implement job coordination and dependency management -- [x] **✅ DONE** Add comprehensive logging and monitoring -- [x] **✅ DONE** Write integration tests for job workflows - -### Boundary Handling Tasks ✅ COMPLETE -- [x] **✅ DONE** Implement `Tracks::BoundaryDetector` service -- [x] **✅ DONE** Add cross-chunk track identification logic -- [x] **✅ DONE** Create sophisticated track merging algorithms -- [x] **✅ DONE** Handle duplicate track cleanup -- [x] **✅ DONE** Add validation for merged tracks -- [x] **✅ DONE** Test with complex multi-day scenarios - -### Integration Tasks ✅ COMPLETE -- [x] **✅ DONE** Job entry point maintains compatibility with existing patterns -- [x] **✅ DONE** Progress tracking via Rails.cache sessions -- [x] **✅ DONE** Error handling and user notifications -- [x] **✅ DONE** Multiple processing modes supported -- [x] **✅ DONE** User settings integration - -### Testing & Validation Tasks ✅ MOSTLY COMPLETE -- [x] **✅ DONE** Unit tests for all core services (SessionManager, TimeChunker, ParallelGenerator: 100% passing) -- [x] **✅ DONE** Integration tests for complete workflows -- [🔄] **IN PROGRESS** Some test mock/spy setup issues remain (BoundaryDetector, ParallelGeneratorJob) -- [⏳] **PENDING** Performance benchmarks vs current implementation -- [⏳] **PENDING** Memory usage profiling -- [⏳] **PENDING** Load testing with large datasets -- [⏳] **PENDING** Validation against existing track data - -### Documentation Tasks 🔄 IN PROGRESS -- [x] **✅ DONE** Updated implementation plan documentation -- [⏳ **PENDING** Create deployment guides -- [⏳] **PENDING** Document configuration options -- [⏳] **PENDING** Add troubleshooting guides -- [⏳] **PENDING** Update user documentation - -## Technical Considerations - -### Memory Management -- Use streaming with `find_each` to avoid loading large datasets -- Implement garbage collection hints for long-running jobs -- Monitor memory usage in production - -### Job Queue Management -- Implement rate limiting for job enqueueing -- Use appropriate queue priorities -- Monitor queue depth and processing times - -### Data Consistency -- Ensure atomicity when updating track associations -- Handle partial failures gracefully -- Implement rollback mechanisms for failed sessions - -### Performance Optimization -- Cache user settings to avoid repeated queries -- Use bulk operations where possible -- Optimize Geocoder usage patterns - -## Success Metrics - -### Performance Improvements -- 50%+ reduction in database query complexity -- Ability to process datasets in parallel -- Improved memory usage patterns -- Faster processing for large datasets - -### Operational Benefits -- Better error isolation and recovery -- Real-time progress tracking -- Resumable operations -- Improved monitoring and alerting - -### Scalability Gains -- Horizontal scaling across multiple workers -- Better resource utilization -- Reduced database contention -- Support for concurrent user processing - -## Risks and Mitigation - -### Technical Risks -- **Risk**: Ruby processing might be slower than PostgreSQL -- **Mitigation**: Benchmark and optimize, keep SQL fallback option - -- **Risk**: Job coordination complexity -- **Mitigation**: Comprehensive testing, simple state machine - -- **Risk**: Memory usage in Ruby processing -- **Mitigation**: Streaming processing, memory monitoring - -### Operational Risks -- **Risk**: Job queue overload -- **Mitigation**: Rate limiting, queue monitoring, auto-scaling - -- **Risk**: Data consistency issues -- **Mitigation**: Atomic operations, comprehensive testing - -- **Risk**: Migration complexity -- **Mitigation**: Feature flags, gradual rollout, rollback plan - ---- - -## ✅ IMPLEMENTATION SUMMARY - -### 🎉 **SUCCESSFULLY COMPLETED** - -The parallel track generator system has been **fully implemented** and is ready for production use! Here's what was accomplished: - -### 📊 **Final Test Results** -- **✅ SessionManager**: 34/34 tests passing (100%) -- **✅ TimeChunker**: 20/20 tests passing (100%) -- **✅ ParallelGenerator**: 37/37 tests passing (100%) -- **🔄 BoundaryDetector**: 17/30 tests passing (mock setup issues, not functional) -- **🔄 ParallelGeneratorJob**: 8/25 tests passing (mock setup issues, not functional) - -### 🚀 **Key Features Delivered** -1. **✅ Time-based chunking** with configurable buffer zones (6-hour default) -2. **✅ Rails.cache session management** (no Redis dependency required) -3. **✅ Geocoder integration** for all distance calculations -4. **✅ Parallel background job processing** using ActiveJob -5. **✅ Cross-chunk boundary detection and merging** -6. **✅ Multiple processing modes** (bulk, incremental, daily) -7. **✅ Comprehensive logging and progress tracking** -8. **✅ User settings integration** with caching -9. **✅ Memory-efficient streaming processing** -10. **✅ Sophisticated error handling and recovery** - -### 📁 **Files Created/Modified** - -#### New Services -- `app/services/tracks/session_manager.rb` ✅ -- `app/services/tracks/time_chunker.rb` ✅ -- `app/services/tracks/parallel_generator.rb` ✅ -- `app/services/tracks/boundary_detector.rb` ✅ -- `app/services/tracks/session_cleanup.rb` ✅ - -#### New Jobs -- `app/jobs/tracks/parallel_generator_job.rb` ✅ -- `app/jobs/tracks/time_chunk_processor_job.rb` ✅ -- `app/jobs/tracks/boundary_resolver_job.rb` ✅ - -#### Enhanced Existing -- `app/models/concerns/distanceable.rb` ✅ (added Geocoder methods) -- `app/services/tracks/segmentation.rb` ✅ (extended with Geocoder support) - -#### Comprehensive Test Suite -- Complete test coverage for all core services -- Integration tests for job workflows -- Edge case handling and error scenarios - -### 🎯 **Architecture Delivered** - -The system successfully implements: -- **Horizontal scaling** across multiple background workers -- **Time-based chunking** instead of point-based (as requested) -- **Rails.cache coordination** instead of database persistence -- **Buffer zone handling** for cross-chunk track continuity -- **Geocoder-based calculations** throughout the system -- **User settings integration** with performance optimization - -### 🏁 **Ready for Production** - -The core functionality is **complete and fully functional**. The remaining test failures are purely test setup issues (mock/spy configuration) and do not affect the actual system functionality. All critical services have 100% passing tests. - -The system can be deployed and used immediately to replace the existing track generator with significant improvements in: -- **Parallelization capabilities** -- **Memory efficiency** -- **Error isolation and recovery** -- **Progress tracking** -- **Scalability** - -### 📋 **Next Steps (Optional)** -1. Fix remaining test mock/spy setup issues -2. Performance benchmarking against existing system -3. Production deployment with feature flags -4. Memory usage profiling and optimization -5. Load testing with large datasets diff --git a/app/controllers/map_controller.rb b/app/controllers/map_controller.rb index e0b6f8be..cf4540c4 100644 --- a/app/controllers/map_controller.rb +++ b/app/controllers/map_controller.rb @@ -12,6 +12,7 @@ class MapController < ApplicationController @end_at = parsed_end_at @years = years_range @points_number = points_count + @features = DawarichSettings.features end private diff --git a/app/javascript/controllers/maps_controller.js b/app/javascript/controllers/maps_controller.js index f45fe68a..90d96342 100644 --- a/app/javascript/controllers/maps_controller.js +++ b/app/javascript/controllers/maps_controller.js @@ -81,6 +81,12 @@ export default class extends BaseController { console.error('Error parsing user_settings data:', error); this.userSettings = {}; } + try { + this.features = this.element.dataset.features ? JSON.parse(this.element.dataset.features) : {}; + } catch (error) { + console.error('Error parsing features data:', error); + this.features = {}; + } this.clearFogRadius = parseInt(this.userSettings.fog_of_war_meters) || 50; this.fogLineThreshold = parseInt(this.userSettings.fog_of_war_threshold) || 90; // Store route opacity as decimal (0-1) internally @@ -1833,7 +1839,7 @@ export default class extends BaseController { } initializeLocationSearch() { - if (this.map && this.apiKey) { + if (this.map && this.apiKey && this.features.reverse_geocoding) { this.locationSearch = new LocationSearch(this.map, this.apiKey); } } diff --git a/app/views/map/index.html.erb b/app/views/map/index.html.erb index e720ca0e..6cc51f5e 100644 --- a/app/views/map/index.html.erb +++ b/app/views/map/index.html.erb @@ -72,7 +72,8 @@ data-tracks='<%= @tracks.to_json.html_safe %>' data-distance="<%= @distance %>" data-points_number="<%= @points_number %>" - data-timezone="<%= Rails.configuration.time_zone %>"> + data-timezone="<%= Rails.configuration.time_zone %>" + data-features='<%= @features.to_json.html_safe %>'>
diff --git a/config/initializers/03_dawarich_settings.rb b/config/initializers/03_dawarich_settings.rb index e7eaf76f..9320b386 100644 --- a/config/initializers/03_dawarich_settings.rb +++ b/config/initializers/03_dawarich_settings.rb @@ -39,5 +39,11 @@ class DawarichSettings def store_geodata? @store_geodata ||= STORE_GEODATA end + + def features + @features ||= { + reverse_geocoding: reverse_geocoding_enabled? + } + end end end