From cba793354c5119a1424595a1beb4c53043f8858c Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Sat, 29 Nov 2025 15:18:34 +0100 Subject: [PATCH] 489 feature request infer missing gps data between close points in visit computation (#509) --- .github/workflows/ci.yml | 248 ++++-- pom.xml | 19 + .../reitti/config/CustomOidcUserService.java | 2 +- .../reitti/config/LocationDensityConfig.java | 39 + .../reitti/config/RabbitMQConfig.java | 45 - .../controller/api/IngestApiController.java | 4 +- .../settings/GeoCodingSettingsController.java | 7 +- .../settings/PlacesSettingsController.java | 4 +- .../SettingsVisitSensitivityController.java | 2 +- .../TransportationModesController.java | 3 +- .../reitti/dto/ConfigurationForm.java | 39 +- .../reitti/event/LocationDataEvent.java | 18 +- .../reitti/event/LocationProcessEvent.java | 23 +- .../event/ProcessedVisitCreatedEvent.java | 19 +- .../reitti/event/RecalculateTripEvent.java | 16 +- .../event/SignificantPlaceCreatedEvent.java | 30 +- .../reitti/event/TriggerProcessingEvent.java | 20 +- .../reitti/event/VisitCreatedEvent.java | 22 - .../reitti/event/VisitUpdatedEvent.java | 32 - .../reitti/model/geo/GeoUtils.java | 12 +- .../reitti/model/geo/RawLocationPoint.java | 36 +- .../model/processing/DetectionParameter.java | 47 +- .../reitti/repository/MemoryJdbcService.java | 24 +- .../repository/OptimisticLockException.java | 2 +- .../PreviewRawLocationPointJdbcService.java | 14 +- .../repository/PreviewTripJdbcService.java | 20 +- ...ewVisitDetectionParametersJdbcService.java | 9 +- .../repository/PreviewVisitJdbcService.java | 2 +- .../RawLocationPointJdbcService.java | 178 +++- .../SignificantPlaceOverrideJdbcService.java | 4 +- .../reitti/repository/TripJdbcService.java | 49 +- .../VisitDetectionParametersJdbcService.java | 27 +- .../reitti/repository/VisitJdbcService.java | 15 +- .../reitti/service/ImportBatchProcessor.java | 46 +- .../service/MessageDispatcherService.java | 59 +- .../reitti/service/QueueStatsService.java | 6 - .../reitti/service/StorageService.java | 7 + .../reitti/service/UserService.java | 3 +- .../service/VisitDetectionPreviewService.java | 10 +- .../importer/BaseGoogleTimelineImporter.java | 31 +- .../service/importer/GeoJsonImporter.java | 5 +- .../GoogleAndroidTimelineImporter.java | 8 +- .../importer/GoogleIOSTimelineImporter.java | 12 +- .../importer/GoogleRecordsImporter.java | 4 +- .../reitti/service/importer/GpxImporter.java | 4 +- .../OwnTracksRecorderIntegrationService.java | 4 +- .../LocationDataDensityNormalizer.java | 330 +++++++ .../LocationDataIngestPipeline.java | 47 +- .../processing/ProcessingPipelineTrigger.java | 86 +- .../SyntheticLocationPointGenerator.java | 126 +++ .../processing/TripDetectionService.java | 201 ----- .../UnifiedLocationProcessingService.java | 824 ++++++++++++++++++ .../processing/VisitDetectionService.java | 369 -------- .../processing/VisitMergingService.java | 310 ------- src/main/resources/application-ci.properties | 5 + src/main/resources/application-dev.properties | 2 +- src/main/resources/application.properties | 4 + ...ignored_columns_to_raw_location_points.sql | 27 + ...nstraint_to_visit_detection_parameters.sql | 23 + .../fragments/configuration-form.html | 22 +- .../fragments/configuration-preview.html | 6 +- .../dedicatedcode/reitti/IntegrationTest.java | 2 +- .../com/dedicatedcode/reitti/TestUtils.java | 6 +- .../dedicatedcode/reitti/TestingService.java | 76 +- .../MemoryControllerTimezoneTest.java | 13 +- .../UserSettingsControllerTest.java | 1 - .../reitti/model/GeoUtilsTest.java | 24 +- .../repository/MemoryJdbcServiceTest.java | 74 +- ...gnificantPlaceOverrideJdbcServiceTest.java | 25 +- .../TransportModeJdbcServiceTest.java | 12 - .../UserJdbcServiceIntegrationTest.java | 18 +- ...sitDetectionParametersJdbcServiceTest.java | 113 ++- .../DefaultGeocodeServiceManagerTest.java | 12 +- .../BaseGoogleTimelineImporterTest.java | 22 +- .../service/importer/GeoJsonImporterTest.java | 16 +- .../GoogleAndroidTimelineImporterTest.java | 14 +- .../GoogleIOSTimelineImporterTest.java | 16 +- .../importer/GoogleRecordsImporterTest.java | 14 +- .../LocationDataDensityNormalizerTest.java | 174 ++++ .../LocationDataIngestPipelineTest.java | 35 - .../processing/ProcessingPipelineTest.java | 123 +-- .../SyntheticLocationPointGeneratorTest.java | 201 +++++ .../processing/VisitDetectionServiceTest.java | 19 +- .../resources/application-test.properties | 5 +- 84 files changed, 2789 insertions(+), 1838 deletions(-) create mode 100644 src/main/java/com/dedicatedcode/reitti/config/LocationDensityConfig.java delete mode 100644 src/main/java/com/dedicatedcode/reitti/event/VisitCreatedEvent.java delete mode 100644 src/main/java/com/dedicatedcode/reitti/event/VisitUpdatedEvent.java create mode 100644 src/main/java/com/dedicatedcode/reitti/service/processing/LocationDataDensityNormalizer.java create mode 100644 src/main/java/com/dedicatedcode/reitti/service/processing/SyntheticLocationPointGenerator.java delete mode 100644 src/main/java/com/dedicatedcode/reitti/service/processing/TripDetectionService.java create mode 100644 src/main/java/com/dedicatedcode/reitti/service/processing/UnifiedLocationProcessingService.java delete mode 100644 src/main/java/com/dedicatedcode/reitti/service/processing/VisitDetectionService.java delete mode 100644 src/main/java/com/dedicatedcode/reitti/service/processing/VisitMergingService.java create mode 100644 src/main/resources/application-ci.properties create mode 100644 src/main/resources/db/migration/V65__add_synthetic_ignored_columns_to_raw_location_points.sql create mode 100644 src/main/resources/db/migration/V66__add_unique_constraint_to_visit_detection_parameters.sql create mode 100644 src/test/java/com/dedicatedcode/reitti/service/processing/LocationDataDensityNormalizerTest.java delete mode 100644 src/test/java/com/dedicatedcode/reitti/service/processing/LocationDataIngestPipelineTest.java create mode 100644 src/test/java/com/dedicatedcode/reitti/service/processing/SyntheticLocationPointGeneratorTest.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1e56412..0d8a6079 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,120 +6,176 @@ on: - main jobs: - test: + build: runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@v4 - - name: Set up JDK 25 - uses: actions/setup-java@v4 - with: - java-version: '25' - distribution: 'temurin' - cache: maven + - name: Set up JDK 25 + uses: actions/setup-java@v4 + with: + java-version: '25' + distribution: 'temurin' + cache: maven - - name: Install dependencies for acknowledgments script - run: | - sudo apt-get update - sudo apt-get install -y jq curl + - name: Install dependencies for acknowledgments script + run: | + sudo apt-get update + sudo apt-get install -y jq curl - - name: Generate acknowledgments data - run: | - chmod +x scripts/generate-acknowledgments.sh - ./scripts/generate-acknowledgments.sh - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Generate acknowledgments data + run: | + chmod +x scripts/generate-acknowledgments.sh + ./scripts/generate-acknowledgments.sh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Build - run: mvn verify -DskipTests + - name: Build JAR without tests + run: mvn compile package -DskipTests - - name: Create bundle - run: mkdir staging && cp target/*.jar staging + - name: Create bundle + run: mkdir staging && cp target/*.jar staging - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Upload JAR artifact + uses: actions/upload-artifact@v4 + with: + name: jar-artifact + path: staging/ + retention-days: 1 - - name: Cache Docker layers - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache - key: docker-${{ runner.os }}-${{ github.sha }} - restore-keys: | - docker-${{ runner.os }}- + unit-tests: + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout code + uses: actions/checkout@v4 - - name: Build docker image - uses: docker/build-push-action@v6 - with: - platforms: linux/amd64 - context: . - push: false - load: true - tags: dedicatedcode/reitti:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + - name: Set up JDK 25 + uses: actions/setup-java@v4 + with: + java-version: '25' + distribution: 'temurin' + cache: maven - - name: Move Docker cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - - name: Start docker-compose - run: docker compose -f docker-compose.ci.yml up -d - working-directory: e2e + - name: Run unit tests + run: mvn test + env: + DOCKER_HOST: unix:///var/run/docker.sock + SPRING_PROFILES_ACTIVE: test,ci - - name: Wait for app to be ready - run: | - timeout 60 bash -c 'until curl -f http://localhost:8080; do sleep 2; done' - working-directory: e2e + - name: Upload coverage reports as artifact + uses: actions/upload-artifact@v4 + with: + name: coverage-reports + path: | + target/site/jacoco/ + retention-days: 30 - - name: Cache npm dependencies - uses: actions/cache@v4 - with: - path: e2e/node_modules - key: npm-${{ runner.os }}-${{ hashFiles('e2e/package-lock.json') }} - restore-keys: | - npm-${{ runner.os }}- + e2e-tests: + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout code + uses: actions/checkout@v4 - - name: Install dependencies - run: npm ci - working-directory: e2e + - name: Download JAR artifact + uses: actions/download-artifact@v4 + with: + name: jar-artifact + path: staging/ - - name: Cache Playwright browsers - uses: actions/cache@v4 - with: - path: ~/.cache/ms-playwright - key: playwright-${{ runner.os }}-${{ hashFiles('e2e/package-lock.json') }} - restore-keys: | - playwright-${{ runner.os }}- + - name: Copy JAR to target directory for Docker build + run: | + mkdir -p target + cp staging/*.jar target/ - - name: Install Playwright browsers - run: npx playwright install --with-deps - working-directory: e2e + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - - name: Run Playwright tests - run: CI=1 npx playwright test --project=chromium --project=firefox --project=webkit - env: - BASE_URL: http://localhost:8080 - working-directory: e2e + - name: Cache Docker layers + uses: actions/cache@v4 + with: + path: /tmp/.buildx-cache + key: docker-${{ runner.os }}-${{ github.sha }} + restore-keys: | + docker-${{ runner.os }}- - - name: Upload Playwright test results - uses: actions/upload-artifact@v4 - if: failure() - with: - name: playwright-test-results - path: e2e/test-results/ - retention-days: 30 + - name: Build docker image + uses: docker/build-push-action@v6 + with: + platforms: linux/amd64 + context: . + push: false + load: true + tags: dedicatedcode/reitti:latest + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - - name: Upload Playwright report - uses: actions/upload-artifact@v4 - if: failure() - with: - name: playwright-report - path: e2e/playwright-report/ - retention-days: 30 + - name: Move Docker cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - name: Stop docker-compose - run: docker compose -f docker-compose.ci.yml down - working-directory: e2e - if: always() + - name: Start docker-compose + run: docker compose -f docker-compose.ci.yml up -d + working-directory: e2e + + - name: Wait for app to be ready + run: | + timeout 60 bash -c 'until curl -f http://localhost:8080; do sleep 2; done' + working-directory: e2e + + - name: Cache npm dependencies + uses: actions/cache@v4 + with: + path: e2e/node_modules + key: npm-${{ runner.os }}-${{ hashFiles('e2e/package-lock.json') }} + restore-keys: | + npm-${{ runner.os }}- + + - name: Install dependencies + run: npm ci + working-directory: e2e + + - name: Cache Playwright browsers + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: playwright-${{ runner.os }}-${{ hashFiles('e2e/package-lock.json') }} + restore-keys: | + playwright-${{ runner.os }}- + + - name: Install Playwright browsers + run: npx playwright install --with-deps + working-directory: e2e + + - name: Run Playwright tests + run: CI=1 npx playwright test --project=chromium --project=firefox --project=webkit + env: + BASE_URL: http://localhost:8080 + working-directory: e2e + + - name: Upload Playwright test results + uses: actions/upload-artifact@v4 + if: failure() + with: + name: playwright-test-results + path: e2e/test-results/ + retention-days: 30 + + - name: Upload Playwright report + uses: actions/upload-artifact@v4 + if: failure() + with: + name: playwright-report + path: e2e/playwright-report/ + retention-days: 30 + + - name: Stop docker-compose + run: docker compose -f docker-compose.ci.yml down + working-directory: e2e + if: always() diff --git a/pom.xml b/pom.xml index 25480558..098c66c1 100644 --- a/pom.xml +++ b/pom.xml @@ -200,6 +200,25 @@ full + + org.jacoco + jacoco-maven-plugin + 0.8.14 + + + + prepare-agent + + + + report + test + + report + + + + diff --git a/src/main/java/com/dedicatedcode/reitti/config/CustomOidcUserService.java b/src/main/java/com/dedicatedcode/reitti/config/CustomOidcUserService.java index 158ef9f9..164a5070 100644 --- a/src/main/java/com/dedicatedcode/reitti/config/CustomOidcUserService.java +++ b/src/main/java/com/dedicatedcode/reitti/config/CustomOidcUserService.java @@ -141,7 +141,7 @@ public class CustomOidcUserService extends OidcUserService { log.warn("No avatar data received from URL: {}", avatarUrl); } } catch (Exception e) { - log.warn("Failed to download avatar from URL: {} for user ID: {}", avatarUrl, userId, e); + log.warn("Failed to download avatar from URL: {} for user ID: {}. {}", avatarUrl, userId, e.getMessage()); } } diff --git a/src/main/java/com/dedicatedcode/reitti/config/LocationDensityConfig.java b/src/main/java/com/dedicatedcode/reitti/config/LocationDensityConfig.java new file mode 100644 index 00000000..15602b1d --- /dev/null +++ b/src/main/java/com/dedicatedcode/reitti/config/LocationDensityConfig.java @@ -0,0 +1,39 @@ +package com.dedicatedcode.reitti.config; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class LocationDensityConfig { + + @Value("${reitti.location.density.target-points-per-minute:4}") + private int targetPointsPerMinute; + + public int getTargetPointsPerMinute() { + return targetPointsPerMinute; + } + + /** + * Calculate the target interval in seconds between points + * @return seconds between points (e.g., 15 seconds for 4 points per minute) + */ + public int getTargetIntervalSeconds() { + return 60 / targetPointsPerMinute; + } + + /** + * Calculate the tolerance window in seconds (half the target interval) + * @return tolerance in seconds + */ + public int getToleranceSeconds() { + return getTargetIntervalSeconds() / 2; + } + + /** + * Calculate the gap threshold - gaps larger than this need synthetic points + * @return gap threshold in seconds (2x target interval) + */ + public int getGapThresholdSeconds() { + return getTargetIntervalSeconds() * 2; + } +} diff --git a/src/main/java/com/dedicatedcode/reitti/config/RabbitMQConfig.java b/src/main/java/com/dedicatedcode/reitti/config/RabbitMQConfig.java index 85f2810f..d799dffc 100644 --- a/src/main/java/com/dedicatedcode/reitti/config/RabbitMQConfig.java +++ b/src/main/java/com/dedicatedcode/reitti/config/RabbitMQConfig.java @@ -14,15 +14,9 @@ public class RabbitMQConfig { public static final String EXCHANGE_NAME = "reitti-exchange"; public static final String LOCATION_DATA_QUEUE = "reitti.location.data.v2"; public static final String LOCATION_DATA_ROUTING_KEY = "reitti.location.data.v2"; - public static final String STAY_DETECTION_QUEUE = "reitti.visit.detection.v2"; - public static final String STAY_DETECTION_ROUTING_KEY = "reitti.visit.created.v2"; - public static final String MERGE_VISIT_QUEUE = "reitti.visit.merge.v2"; - public static final String MERGE_VISIT_ROUTING_KEY = "reitti.visit.merged.v2"; public static final String SIGNIFICANT_PLACE_QUEUE = "reitti.place.created.v2"; public static final String SIGNIFICANT_PLACE_ROUTING_KEY = "reitti.place.created.v2"; - public static final String DETECT_TRIP_QUEUE = "reitti.trip.created.v2"; public static final String RECALCULATE_TRIP_QUEUE = "reitti.trip.recalculate.v2"; - public static final String DETECT_TRIP_ROUTING_KEY = "reitti.trip.created.v2"; public static final String DETECT_TRIP_RECALCULATION_ROUTING_KEY = "reitti.trip.recalculate.v2"; public static final String TRIGGER_PROCESSING_PIPELINE_QUEUE = "reitti.processing.v2"; public static final String TRIGGER_PROCESSING_PIPELINE_ROUTING_KEY = "reitti.processing.start.v2"; @@ -53,14 +47,6 @@ public class RabbitMQConfig { .build(); } - @Bean - public Queue detectTripQueue() { - return QueueBuilder.durable(DETECT_TRIP_QUEUE) - .withArgument("x-dead-letter-exchange", DLX_NAME) - .withArgument("x-dead-letter-routing-key", DLQ_NAME) - .build(); - } - @Bean public Queue recaluclateTripQueue() { return QueueBuilder.durable(RECALCULATE_TRIP_QUEUE) @@ -69,14 +55,6 @@ public class RabbitMQConfig { .build(); } - @Bean - public Queue mergeVisitQueue() { - return QueueBuilder.durable(MERGE_VISIT_QUEUE) - .withArgument("x-dead-letter-exchange", DLX_NAME) - .withArgument("x-dead-letter-routing-key", DLQ_NAME) - .build(); - } - @Bean public Queue significantPlaceQueue() { return QueueBuilder.durable(SIGNIFICANT_PLACE_QUEUE) @@ -85,14 +63,6 @@ public class RabbitMQConfig { .build(); } - @Bean - public Queue stayDetectionQueue() { - return QueueBuilder.durable(STAY_DETECTION_QUEUE) - .withArgument("x-dead-letter-exchange", DLX_NAME) - .withArgument("x-dead-letter-routing-key", DLQ_NAME) - .build(); - } - @Bean public Queue triggerProcessingQueue() { return QueueBuilder.nonDurable(TRIGGER_PROCESSING_PIPELINE_QUEUE) @@ -119,26 +89,11 @@ public class RabbitMQConfig { return BindingBuilder.bind(significantPlaceQueue).to(exchange).with(SIGNIFICANT_PLACE_ROUTING_KEY); } - @Bean - public Binding mergeVisitBinding(Queue mergeVisitQueue, TopicExchange exchange) { - return BindingBuilder.bind(mergeVisitQueue).to(exchange).with(MERGE_VISIT_ROUTING_KEY); - } - - @Bean - public Binding detectTripBinding(Queue detectTripQueue, TopicExchange exchange) { - return BindingBuilder.bind(detectTripQueue).to(exchange).with(DETECT_TRIP_ROUTING_KEY); - } - @Bean public Binding recalculateTripBinding(Queue recaluclateTripQueue , TopicExchange exchange) { return BindingBuilder.bind(recaluclateTripQueue).to(exchange).with(DETECT_TRIP_RECALCULATION_ROUTING_KEY); } - @Bean - public Binding stayDetectionBinding(Queue stayDetectionQueue, TopicExchange exchange) { - return BindingBuilder.bind(stayDetectionQueue).to(exchange).with(STAY_DETECTION_ROUTING_KEY); - } - @Bean public Binding triggerProcessingBinding(Queue triggerProcessingQueue, TopicExchange exchange) { return BindingBuilder.bind(triggerProcessingQueue).to(exchange).with(TRIGGER_PROCESSING_PIPELINE_ROUTING_KEY); diff --git a/src/main/java/com/dedicatedcode/reitti/controller/api/IngestApiController.java b/src/main/java/com/dedicatedcode/reitti/controller/api/IngestApiController.java index ec8de348..b003cae3 100644 --- a/src/main/java/com/dedicatedcode/reitti/controller/api/IngestApiController.java +++ b/src/main/java/com/dedicatedcode/reitti/controller/api/IngestApiController.java @@ -68,7 +68,7 @@ public class IngestApiController { return ResponseEntity.ok(Map.of()); } - this.batchProcessor.sendToQueue(user, Collections.singletonList(locationPoint)); + this.batchProcessor.processBatch(user, Collections.singletonList(locationPoint)); logger.debug("Successfully received and queued Owntracks location point for user {}", user.getUsername()); @@ -108,7 +108,7 @@ public class IngestApiController { )); } - this.batchProcessor.sendToQueue(user, locationPoints); + this.batchProcessor.processBatch(user, locationPoints); logger.debug("Successfully received and queued {} Overland location points for user {}", locationPoints.size(), user.getUsername()); diff --git a/src/main/java/com/dedicatedcode/reitti/controller/settings/GeoCodingSettingsController.java b/src/main/java/com/dedicatedcode/reitti/controller/settings/GeoCodingSettingsController.java index 170133bc..2b8aacaf 100644 --- a/src/main/java/com/dedicatedcode/reitti/controller/settings/GeoCodingSettingsController.java +++ b/src/main/java/com/dedicatedcode/reitti/controller/settings/GeoCodingSettingsController.java @@ -22,6 +22,7 @@ import org.springframework.ui.Model; import org.springframework.web.bind.annotation.*; import java.util.List; +import java.util.UUID; @Controller @RequestMapping("/settings/geocode-services") @@ -140,7 +141,8 @@ public class GeoCodingSettingsController { null, place.getId(), place.getLatitudeCentroid(), - place.getLongitudeCentroid() + place.getLongitudeCentroid(), + UUID.randomUUID().toString() ); rabbitTemplate.convertAndSend(RabbitMQConfig.EXCHANGE_NAME, RabbitMQConfig.SIGNIFICANT_PLACE_ROUTING_KEY, event); } @@ -183,7 +185,8 @@ public class GeoCodingSettingsController { null, place.getId(), place.getLatitudeCentroid(), - place.getLongitudeCentroid() + place.getLongitudeCentroid(), + UUID.randomUUID().toString() ); rabbitTemplate.convertAndSend(RabbitMQConfig.EXCHANGE_NAME, RabbitMQConfig.SIGNIFICANT_PLACE_ROUTING_KEY, event); } diff --git a/src/main/java/com/dedicatedcode/reitti/controller/settings/PlacesSettingsController.java b/src/main/java/com/dedicatedcode/reitti/controller/settings/PlacesSettingsController.java index 4ec35b83..d4dc944e 100644 --- a/src/main/java/com/dedicatedcode/reitti/controller/settings/PlacesSettingsController.java +++ b/src/main/java/com/dedicatedcode/reitti/controller/settings/PlacesSettingsController.java @@ -26,6 +26,7 @@ import org.springframework.web.bind.annotation.*; import org.springframework.web.server.ResponseStatusException; import java.util.List; +import java.util.UUID; import java.util.stream.Collectors; @Controller @@ -201,7 +202,8 @@ public class PlacesSettingsController { null, significantPlace.getId(), significantPlace.getLatitudeCentroid(), - significantPlace.getLongitudeCentroid() + significantPlace.getLongitudeCentroid(), + UUID.randomUUID().toString() ); rabbitTemplate.convertAndSend(RabbitMQConfig.EXCHANGE_NAME, RabbitMQConfig.SIGNIFICANT_PLACE_ROUTING_KEY, event); diff --git a/src/main/java/com/dedicatedcode/reitti/controller/settings/SettingsVisitSensitivityController.java b/src/main/java/com/dedicatedcode/reitti/controller/settings/SettingsVisitSensitivityController.java index f8efc56f..de0f2dd5 100644 --- a/src/main/java/com/dedicatedcode/reitti/controller/settings/SettingsVisitSensitivityController.java +++ b/src/main/java/com/dedicatedcode/reitti/controller/settings/SettingsVisitSensitivityController.java @@ -276,7 +276,7 @@ public class SettingsVisitSensitivityController { rawLocationPointJdbcService.markAllAsUnprocessedForUser(user); allConfigurationsForUser.forEach(config -> this.configurationService.updateConfiguration(config.withRecalculationState(RecalculationState.DONE))); log.debug("Starting recalculation of all configurations"); - processingPipelineTrigger.start(); + processingPipelineTrigger.start(user); } catch (Exception e) { log.error("Error clearing time range", e); } diff --git a/src/main/java/com/dedicatedcode/reitti/controller/settings/TransportationModesController.java b/src/main/java/com/dedicatedcode/reitti/controller/settings/TransportationModesController.java index 337a31be..f1524412 100644 --- a/src/main/java/com/dedicatedcode/reitti/controller/settings/TransportationModesController.java +++ b/src/main/java/com/dedicatedcode/reitti/controller/settings/TransportationModesController.java @@ -22,6 +22,7 @@ import org.springframework.web.servlet.mvc.support.RedirectAttributes; import java.util.Arrays; import java.util.List; import java.util.Objects; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; @@ -199,7 +200,7 @@ public class TransportationModesController { tripJdbcService.findIdsByUser(user).forEach(tripId -> { rabbitTemplate.convertAndSend(RabbitMQConfig.EXCHANGE_NAME, RabbitMQConfig.DETECT_TRIP_RECALCULATION_ROUTING_KEY, - new RecalculateTripEvent(user.getUsername(), tripId)); + new RecalculateTripEvent(user.getUsername(), tripId, UUID.randomUUID().toString())); }); }); diff --git a/src/main/java/com/dedicatedcode/reitti/dto/ConfigurationForm.java b/src/main/java/com/dedicatedcode/reitti/dto/ConfigurationForm.java index e7b627f2..aeac3ea6 100644 --- a/src/main/java/com/dedicatedcode/reitti/dto/ConfigurationForm.java +++ b/src/main/java/com/dedicatedcode/reitti/dto/ConfigurationForm.java @@ -14,8 +14,6 @@ public class ConfigurationForm { private Integer sensitivityLevel = 3; // 1-5 scale // Advanced mode - all Configuration fields - private Long searchDistanceInMeters; - private Integer minimumAdjacentPoints; private Long minimumStayTimeInSeconds; private Long maxMergeTimeBetweenSameStayPoints; private Long searchDurationInHours; @@ -35,12 +33,6 @@ public class ConfigurationForm { public Integer getSensitivityLevel() { return sensitivityLevel; } public void setSensitivityLevel(Integer sensitivityLevel) { this.sensitivityLevel = sensitivityLevel; } - public Long getSearchDistanceInMeters() { return searchDistanceInMeters; } - public void setSearchDistanceInMeters(Long searchDistanceInMeters) { this.searchDistanceInMeters = searchDistanceInMeters; } - - public Integer getMinimumAdjacentPoints() { return minimumAdjacentPoints; } - public void setMinimumAdjacentPoints(Integer minimumAdjacentPoints) { this.minimumAdjacentPoints = minimumAdjacentPoints; } - public Long getMinimumStayTimeInSeconds() { return minimumStayTimeInSeconds; } public void setMinimumStayTimeInSeconds(Long minimumStayTimeInSeconds) { this.minimumStayTimeInSeconds = minimumStayTimeInSeconds; } @@ -65,8 +57,6 @@ public class ConfigurationForm { form.setId(config.getId()); // Set advanced mode values - form.setSearchDistanceInMeters(config.getVisitDetection().getSearchDistanceInMeters()); - form.setMinimumAdjacentPoints(config.getVisitDetection().getMinimumAdjacentPoints()); form.setMinimumStayTimeInSeconds(config.getVisitDetection().getMinimumStayTimeInSeconds()); form.setMaxMergeTimeBetweenSameStayPoints(config.getVisitDetection().getMaxMergeTimeBetweenSameStayPoints()); form.setSearchDurationInHours(config.getVisitMerging().getSearchDurationInHours()); @@ -101,7 +91,7 @@ public class ConfigurationForm { return level; } } - return null; // No match found + return null; } private static boolean configurationMatches(DetectionParameter config, @@ -110,9 +100,7 @@ public class ConfigurationForm { DetectionParameter.VisitDetection actualDetection = config.getVisitDetection(); DetectionParameter.VisitMerging actualMerging = config.getVisitMerging(); - return actualDetection.getSearchDistanceInMeters() == expectedDetection.getSearchDistanceInMeters() && - actualDetection.getMinimumAdjacentPoints() == expectedDetection.getMinimumAdjacentPoints() && - actualDetection.getMinimumStayTimeInSeconds() == expectedDetection.getMinimumStayTimeInSeconds() && + return actualDetection.getMinimumStayTimeInSeconds() == expectedDetection.getMinimumStayTimeInSeconds() && actualDetection.getMaxMergeTimeBetweenSameStayPoints() == expectedDetection.getMaxMergeTimeBetweenSameStayPoints() && actualMerging.getSearchDurationInHours() == expectedMerging.getSearchDurationInHours() && actualMerging.getMaxMergeTimeBetweenSameVisits() == expectedMerging.getMaxMergeTimeBetweenSameVisits() && @@ -121,11 +109,11 @@ public class ConfigurationForm { private static DetectionParameter.VisitDetection mapSensitivityToVisitDetection(int level) { return switch (level) { - case 1 -> new DetectionParameter.VisitDetection(200, 8, 600, 600); // Low sensitivity - case 2 -> new DetectionParameter.VisitDetection(150, 6, 450, 450); - case 3 -> new DetectionParameter.VisitDetection(100, 5, 300, 300); // Medium (baseline) - case 4 -> new DetectionParameter.VisitDetection(75, 4, 225, 225); - case 5 -> new DetectionParameter.VisitDetection(50, 3, 150, 150); // High sensitivity + case 1 -> new DetectionParameter.VisitDetection(600, 600); // Low sensitivity + case 2 -> new DetectionParameter.VisitDetection(450, 450); + case 3 -> new DetectionParameter.VisitDetection(300, 300); // Medium (baseline) + case 4 -> new DetectionParameter.VisitDetection(225, 225); + case 5 -> new DetectionParameter.VisitDetection(150, 150); // High sensitivity default -> throw new IllegalArgumentException("Unhandled level [" + level + "] detected!"); }; } @@ -150,8 +138,6 @@ public class ConfigurationForm { DetectionParameter.VisitDetection visitDetection = mapSensitivityToVisitDetection(level); DetectionParameter.VisitMerging visitMerging = mapSensitivityToVisitMerging(level); - this.searchDistanceInMeters = visitDetection.getSearchDistanceInMeters(); - this.minimumAdjacentPoints = visitDetection.getMinimumAdjacentPoints(); this.minimumStayTimeInSeconds = visitDetection.getMinimumStayTimeInSeconds(); this.maxMergeTimeBetweenSameStayPoints = visitDetection.getMaxMergeTimeBetweenSameStayPoints(); this.searchDurationInHours = visitMerging.getSearchDurationInHours(); @@ -172,9 +158,7 @@ public class ConfigurationForm { DetectionParameter.VisitDetection originalDetection = original.getVisitDetection(); DetectionParameter.VisitDetection currentDetection = current.getVisitDetection(); - if (originalDetection.getSearchDistanceInMeters() != currentDetection.getSearchDistanceInMeters() || - originalDetection.getMinimumAdjacentPoints() != currentDetection.getMinimumAdjacentPoints() || - originalDetection.getMinimumStayTimeInSeconds() != currentDetection.getMinimumStayTimeInSeconds() || + if (originalDetection.getMinimumStayTimeInSeconds() != currentDetection.getMinimumStayTimeInSeconds() || originalDetection.getMaxMergeTimeBetweenSameStayPoints() != currentDetection.getMaxMergeTimeBetweenSameStayPoints()) { return true; } @@ -204,8 +188,6 @@ public class ConfigurationForm { } else { // Use advanced mode values visitDetection = new DetectionParameter.VisitDetection( - searchDistanceInMeters, - minimumAdjacentPoints, minimumStayTimeInSeconds, maxMergeTimeBetweenSameStayPoints ); @@ -216,8 +198,11 @@ public class ConfigurationForm { ); } + // Use default location density parameters for now + DetectionParameter.LocationDensity locationDensity = new DetectionParameter.LocationDensity(500.0, 120); + Instant validSinceInstant = validSince != null ? ZonedDateTime.of(validSince.atStartOfDay(), timezone).toInstant() : null; - return new DetectionParameter(getId(), visitDetection, visitMerging, validSinceInstant, RecalculationState.NEEDED); + return new DetectionParameter(getId(), visitDetection, visitMerging, locationDensity, validSinceInstant, RecalculationState.NEEDED); } } diff --git a/src/main/java/com/dedicatedcode/reitti/event/LocationDataEvent.java b/src/main/java/com/dedicatedcode/reitti/event/LocationDataEvent.java index 43945738..f91ec820 100644 --- a/src/main/java/com/dedicatedcode/reitti/event/LocationDataEvent.java +++ b/src/main/java/com/dedicatedcode/reitti/event/LocationDataEvent.java @@ -11,14 +11,17 @@ import java.util.List; public class LocationDataEvent implements Serializable { private final String username; private final List points; + private final String traceId; private final Instant receivedAt; @JsonCreator public LocationDataEvent( @JsonProperty("username") String username, - @JsonProperty("points") List points) { + @JsonProperty("points") List points, + @JsonProperty("trace-id") String traceId) { this.username = username; this.points = points; + this.traceId = traceId; this.receivedAt = Instant.now(); } @@ -33,4 +36,17 @@ public class LocationDataEvent implements Serializable { public Instant getReceivedAt() { return receivedAt; } + public String getTraceId() { + return traceId; + } + + @Override + public String toString() { + return "LocationDataEvent{" + + "username='" + username + '\'' + + ", points=" + points.size() + + ", traceId='" + traceId + '\'' + + ", receivedAt=" + receivedAt + + '}'; + } } diff --git a/src/main/java/com/dedicatedcode/reitti/event/LocationProcessEvent.java b/src/main/java/com/dedicatedcode/reitti/event/LocationProcessEvent.java index fcf7b58c..74b8a40d 100644 --- a/src/main/java/com/dedicatedcode/reitti/event/LocationProcessEvent.java +++ b/src/main/java/com/dedicatedcode/reitti/event/LocationProcessEvent.java @@ -6,23 +6,27 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.io.Serializable; import java.time.Instant; import java.util.Objects; +import java.util.UUID; public class LocationProcessEvent implements Serializable { private final String username; private final Instant earliest; private final Instant latest; private final String previewId; + private final String traceId; @JsonCreator public LocationProcessEvent( @JsonProperty("username") String username, @JsonProperty("earliest") Instant earliest, @JsonProperty("latest") Instant latest, - @JsonProperty("previewId") String previewId) { + @JsonProperty("previewId") String previewId, + @JsonProperty("trace-id") String traceId) { this.username = username; this.earliest = earliest; this.latest = latest; this.previewId = previewId; + this.traceId = traceId; } public String getUsername() { @@ -41,15 +45,18 @@ public class LocationProcessEvent implements Serializable { return previewId; } - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - LocationProcessEvent that = (LocationProcessEvent) o; - return Objects.equals(username, that.username) && Objects.equals(earliest, that.earliest) && Objects.equals(latest, that.latest); + public String getTraceId() { + return traceId; } @Override - public int hashCode() { - return Objects.hash(username, earliest, latest); + public String toString() { + return "LocationProcessEvent{" + + "username='" + username + '\'' + + ", earliest=" + earliest + + ", latest=" + latest + + ", previewId='" + previewId + '\'' + + ", traceId='" + traceId + '\'' + + '}'; } } diff --git a/src/main/java/com/dedicatedcode/reitti/event/ProcessedVisitCreatedEvent.java b/src/main/java/com/dedicatedcode/reitti/event/ProcessedVisitCreatedEvent.java index 6877a044..ba98fa07 100644 --- a/src/main/java/com/dedicatedcode/reitti/event/ProcessedVisitCreatedEvent.java +++ b/src/main/java/com/dedicatedcode/reitti/event/ProcessedVisitCreatedEvent.java @@ -6,14 +6,17 @@ public class ProcessedVisitCreatedEvent { private final String username; private final long visitId; private final String previewId; + private final String traceId; public ProcessedVisitCreatedEvent( @JsonProperty String username, @JsonProperty long visitId, - @JsonProperty String previewId) { + @JsonProperty String previewId, + @JsonProperty("trace-id") String traceId) { this.username = username; this.visitId = visitId; this.previewId = previewId; + this.traceId = traceId; } public String getUsername() { @@ -27,4 +30,18 @@ public class ProcessedVisitCreatedEvent { public String getPreviewId() { return previewId; } + + public String getTraceId() { + return traceId; + } + + @Override + public String toString() { + return "ProcessedVisitCreatedEvent{" + + "username='" + username + '\'' + + ", visitId=" + visitId + + ", previewId='" + previewId + '\'' + + ", traceId='" + traceId + '\'' + + '}'; + } } diff --git a/src/main/java/com/dedicatedcode/reitti/event/RecalculateTripEvent.java b/src/main/java/com/dedicatedcode/reitti/event/RecalculateTripEvent.java index ce743d47..18fa6947 100644 --- a/src/main/java/com/dedicatedcode/reitti/event/RecalculateTripEvent.java +++ b/src/main/java/com/dedicatedcode/reitti/event/RecalculateTripEvent.java @@ -1,14 +1,19 @@ package com.dedicatedcode.reitti.event; +import com.fasterxml.jackson.annotation.JsonProperty; + import java.io.Serializable; public class RecalculateTripEvent implements Serializable { private final String username; private final long tripId; + private final String traceId; - public RecalculateTripEvent(String username, long tripId) { + public RecalculateTripEvent(String username, long tripId, + @JsonProperty("trace-id") String traceId) { this.username = username; this.tripId = tripId; + this.traceId = traceId; } public String getUsername() { @@ -18,4 +23,13 @@ public class RecalculateTripEvent implements Serializable { public long getTripId() { return tripId; } + + @Override + public String toString() { + return "RecalculateTripEvent{" + + "username='" + username + '\'' + + ", tripId=" + tripId + + ", traceId='" + traceId + '\'' + + '}'; + } } diff --git a/src/main/java/com/dedicatedcode/reitti/event/SignificantPlaceCreatedEvent.java b/src/main/java/com/dedicatedcode/reitti/event/SignificantPlaceCreatedEvent.java index 1c00e53f..eb904517 100644 --- a/src/main/java/com/dedicatedcode/reitti/event/SignificantPlaceCreatedEvent.java +++ b/src/main/java/com/dedicatedcode/reitti/event/SignificantPlaceCreatedEvent.java @@ -1,6 +1,34 @@ package com.dedicatedcode.reitti.event; +import com.fasterxml.jackson.annotation.JsonProperty; + import java.io.Serializable; -public record SignificantPlaceCreatedEvent(String username, String previewId, Long placeId, Double latitude, Double longitude) implements Serializable { +public record SignificantPlaceCreatedEvent(String username, String previewId, Long placeId, Double latitude, + Double longitude, String traceId) implements Serializable { + public SignificantPlaceCreatedEvent(String username, + String previewId, + Long placeId, + Double latitude, + Double longitude, + @JsonProperty("trace-id") String traceId) { + this.username = username; + this.previewId = previewId; + this.placeId = placeId; + this.latitude = latitude; + this.longitude = longitude; + this.traceId = traceId; + } + + @Override + public String toString() { + return "SignificantPlaceCreatedEvent{" + + "username='" + username + '\'' + + ", previewId='" + previewId + '\'' + + ", placeId=" + placeId + + ", latitude=" + latitude + + ", longitude=" + longitude + + ", traceId='" + traceId + '\'' + + '}'; + } } diff --git a/src/main/java/com/dedicatedcode/reitti/event/TriggerProcessingEvent.java b/src/main/java/com/dedicatedcode/reitti/event/TriggerProcessingEvent.java index d088bb1a..7ad57925 100644 --- a/src/main/java/com/dedicatedcode/reitti/event/TriggerProcessingEvent.java +++ b/src/main/java/com/dedicatedcode/reitti/event/TriggerProcessingEvent.java @@ -5,18 +5,22 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.io.Serializable; import java.time.Instant; +import java.util.UUID; public class TriggerProcessingEvent implements Serializable { private final String username; private final String previewId; private final Instant receivedAt; + private final String traceId; @JsonCreator public TriggerProcessingEvent( @JsonProperty("username") String username, - String previewId) { + String previewId, + @JsonProperty("trace-id") String traceId) { this.username = username; this.previewId = previewId; + this.traceId = traceId; this.receivedAt = Instant.now(); } @@ -31,4 +35,18 @@ public class TriggerProcessingEvent implements Serializable { public String getPreviewId() { return this.previewId; } + + public String getTraceId() { + return traceId; + } + + @Override + public String toString() { + return "TriggerProcessingEvent{" + + "username='" + username + '\'' + + ", previewId='" + previewId + '\'' + + ", receivedAt=" + receivedAt + + ", traceId='" + traceId + '\'' + + '}'; + } } diff --git a/src/main/java/com/dedicatedcode/reitti/event/VisitCreatedEvent.java b/src/main/java/com/dedicatedcode/reitti/event/VisitCreatedEvent.java deleted file mode 100644 index daa1cbcc..00000000 --- a/src/main/java/com/dedicatedcode/reitti/event/VisitCreatedEvent.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.dedicatedcode.reitti.event; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class VisitCreatedEvent { - private final String username; - private final long visitId; - public VisitCreatedEvent( - @JsonProperty String username, - @JsonProperty long visitId) { - this.username = username; - this.visitId = visitId; - } - - public String getUsername() { - return username; - } - - public long getVisitId() { - return visitId; - } -} diff --git a/src/main/java/com/dedicatedcode/reitti/event/VisitUpdatedEvent.java b/src/main/java/com/dedicatedcode/reitti/event/VisitUpdatedEvent.java deleted file mode 100644 index 3404f7e1..00000000 --- a/src/main/java/com/dedicatedcode/reitti/event/VisitUpdatedEvent.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.dedicatedcode.reitti.event; - -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.List; - -public class VisitUpdatedEvent { - private final String username; - private final List visitIds; - private final String previewId; - - public VisitUpdatedEvent( - @JsonProperty String username, - @JsonProperty List visitIds, - @JsonProperty String previewId) { - this.username = username; - this.visitIds = visitIds; - this.previewId = previewId; - } - - public String getUsername() { - return username; - } - - public List getVisitIds() { - return visitIds; - } - - public String getPreviewId() { - return previewId; - } -} diff --git a/src/main/java/com/dedicatedcode/reitti/model/geo/GeoUtils.java b/src/main/java/com/dedicatedcode/reitti/model/geo/GeoUtils.java index 6c3bfadb..d53f6928 100644 --- a/src/main/java/com/dedicatedcode/reitti/model/geo/GeoUtils.java +++ b/src/main/java/com/dedicatedcode/reitti/model/geo/GeoUtils.java @@ -19,11 +19,10 @@ public final class GeoUtils { double lonDiff = Math.abs(lon2 - lon1); if (latDiff < 0.01 && lonDiff < 0.01) { // roughly < 1km - // Fast approximation for short distances double avgLat = Math.toRadians((lat1 + lat2) / 2); double latDistance = Math.toRadians(latDiff); double lonDistance = Math.toRadians(lonDiff) * Math.cos(avgLat); - + return EARTH_RADIUS * Math.sqrt(latDistance * latDistance + lonDistance * lonDistance); } @@ -69,14 +68,9 @@ public final class GeoUtils { * @param latitude The latitude at which to calculate the conversion * @return An array where index 0 is the latitude degrees and index 1 is the longitude degrees */ - public static double[] metersToDegreesAtPosition(double meters, double latitude) { - // For latitude: 1 degree = 111,320 meters (approximately constant) - double latitudeDegrees = meters / 111320.0; - + public static double metersToDegreesAtPosition(double meters, double latitude) { // For longitude: 1 degree = 111,320 * cos(latitude) meters (varies with latitude) - double longitudeDegrees = meters / (111320.0 * Math.cos(Math.toRadians(latitude))); - - return new double[] { latitudeDegrees, longitudeDegrees }; + return meters / (111320.0 * Math.cos(Math.toRadians(latitude))); } public static double calculateTripDistance(List points) { diff --git a/src/main/java/com/dedicatedcode/reitti/model/geo/RawLocationPoint.java b/src/main/java/com/dedicatedcode/reitti/model/geo/RawLocationPoint.java index 2d1d2e66..fef5d664 100644 --- a/src/main/java/com/dedicatedcode/reitti/model/geo/RawLocationPoint.java +++ b/src/main/java/com/dedicatedcode/reitti/model/geo/RawLocationPoint.java @@ -16,28 +16,38 @@ public class RawLocationPoint { private final GeoPoint geom; private final boolean processed; + + private final boolean synthetic; + + private final boolean ignored; private final Long version; public RawLocationPoint() { - this(null, null, null, null, null, false, null); + this(null, null, null, null, null, false, false, false, null); } public RawLocationPoint(Instant timestamp, GeoPoint geom, Double accuracyMeters) { - this(null, timestamp, geom, accuracyMeters, null, false, null); + this(null, timestamp, geom, accuracyMeters, null, false, false, false, null); } public RawLocationPoint(Instant timestamp, GeoPoint geom, Double accuracyMeters, Double elevationMeters) { - this(null, timestamp, geom, accuracyMeters, elevationMeters, false, null); + this(null, timestamp, geom, accuracyMeters, elevationMeters, false, false, false, null); } public RawLocationPoint(Long id, Instant timestamp, GeoPoint geom, Double accuracyMeters, Double elevationMeters, boolean processed, Long version) { + this(id, timestamp, geom, accuracyMeters, elevationMeters, processed, false, false, version); + } + + public RawLocationPoint(Long id, Instant timestamp, GeoPoint geom, Double accuracyMeters, Double elevationMeters, boolean processed, boolean synthetic, boolean ignored, Long version) { this.id = id; this.timestamp = timestamp; this.geom = geom; this.accuracyMeters = accuracyMeters; this.elevationMeters = elevationMeters; this.processed = processed; + this.synthetic = synthetic; + this.ignored = ignored; this.version = version; } @@ -72,13 +82,29 @@ public class RawLocationPoint { public boolean isProcessed() { return processed; } + + public boolean isSynthetic() { + return synthetic; + } + + public boolean isIgnored() { + return ignored; + } public RawLocationPoint markProcessed() { - return new RawLocationPoint(this.id, this.timestamp, this.geom, this.accuracyMeters, this.elevationMeters, true, this.version); + return new RawLocationPoint(this.id, this.timestamp, this.geom, this.accuracyMeters, this.elevationMeters, true, this.synthetic, this.ignored, this.version); + } + + public RawLocationPoint markAsSynthetic() { + return new RawLocationPoint(this.id, this.timestamp, this.geom, this.accuracyMeters, this.elevationMeters, this.processed, true, this.ignored, this.version); + } + + public RawLocationPoint markAsIgnored() { + return new RawLocationPoint(this.id, this.timestamp, this.geom, this.accuracyMeters, this.elevationMeters, this.processed, this.synthetic, true, this.version); } public RawLocationPoint withId(Long id) { - return new RawLocationPoint(id, timestamp, geom, accuracyMeters, elevationMeters, processed, version); + return new RawLocationPoint(id, timestamp, geom, accuracyMeters, elevationMeters, processed, synthetic, ignored, version); } public Long getVersion() { diff --git a/src/main/java/com/dedicatedcode/reitti/model/processing/DetectionParameter.java b/src/main/java/com/dedicatedcode/reitti/model/processing/DetectionParameter.java index df648f9f..b02b102f 100644 --- a/src/main/java/com/dedicatedcode/reitti/model/processing/DetectionParameter.java +++ b/src/main/java/com/dedicatedcode/reitti/model/processing/DetectionParameter.java @@ -7,13 +7,15 @@ public class DetectionParameter implements Serializable { private final Long id; private final VisitDetection visitDetection; private final VisitMerging visitMerging; + private final LocationDensity locationDensity; private final Instant validSince; private final RecalculationState recalculationState; - public DetectionParameter(Long id, VisitDetection visitDetection, VisitMerging visitMerging, Instant validSince, RecalculationState recalculationState) { + public DetectionParameter(Long id, VisitDetection visitDetection, VisitMerging visitMerging, LocationDensity locationDensity, Instant validSince, RecalculationState recalculationState) { this.id = id; this.visitDetection = visitDetection; this.visitMerging = visitMerging; + this.locationDensity = locationDensity; this.validSince = validSince; this.recalculationState = recalculationState; } @@ -30,6 +32,10 @@ public class DetectionParameter implements Serializable { return visitMerging; } + public LocationDensity getLocationDensity() { + return locationDensity; + } + public Instant getValidSince() { return validSince; } @@ -39,31 +45,22 @@ public class DetectionParameter implements Serializable { } public DetectionParameter withRecalculationState(RecalculationState recalculationState) { - return new DetectionParameter(this.id, this.visitDetection, this.visitMerging, this.validSince, recalculationState); + return new DetectionParameter(this.id, this.visitDetection, this.visitMerging, this.locationDensity, this.validSince, recalculationState); + } + + public DetectionParameter withLocationDensity(LocationDensity locationDensity) { + return new DetectionParameter(this.id, this.visitDetection, this.visitMerging, locationDensity, this.validSince, this.recalculationState); } public static class VisitDetection implements Serializable { - private final long searchDistanceInMeters; - private final int minimumAdjacentPoints; private final long minimumStayTimeInSeconds; private final long maxMergeTimeBetweenSameStayPoints; - public VisitDetection(long searchDistanceInMeters, int minimumAdjacentPoints, - long minimumStayTimeInSeconds, long maxMergeTimeBetweenSameStayPoints) { - this.searchDistanceInMeters = searchDistanceInMeters; - this.minimumAdjacentPoints = minimumAdjacentPoints; + public VisitDetection(long minimumStayTimeInSeconds, long maxMergeTimeBetweenSameStayPoints) { this.minimumStayTimeInSeconds = minimumStayTimeInSeconds; this.maxMergeTimeBetweenSameStayPoints = maxMergeTimeBetweenSameStayPoints; } - public long getSearchDistanceInMeters() { - return searchDistanceInMeters; - } - - public int getMinimumAdjacentPoints() { - return minimumAdjacentPoints; - } - public long getMinimumStayTimeInSeconds() { return minimumStayTimeInSeconds; } @@ -97,4 +94,22 @@ public class DetectionParameter implements Serializable { return minDistanceBetweenVisits; } } + + public static class LocationDensity implements Serializable { + private final double maxInterpolationDistanceMeters; + private final long maxInterpolationGapMinutes; + + public LocationDensity(double maxInterpolationDistanceMeters, long maxInterpolationGapMinutes) { + this.maxInterpolationDistanceMeters = maxInterpolationDistanceMeters; + this.maxInterpolationGapMinutes = maxInterpolationGapMinutes; + } + + public double getMaxInterpolationDistanceMeters() { + return maxInterpolationDistanceMeters; + } + + public long getMaxInterpolationGapMinutes() { + return maxInterpolationGapMinutes; + } + } } diff --git a/src/main/java/com/dedicatedcode/reitti/repository/MemoryJdbcService.java b/src/main/java/com/dedicatedcode/reitti/repository/MemoryJdbcService.java index 419f5198..790d5a56 100644 --- a/src/main/java/com/dedicatedcode/reitti/repository/MemoryJdbcService.java +++ b/src/main/java/com/dedicatedcode/reitti/repository/MemoryJdbcService.java @@ -13,7 +13,6 @@ import java.sql.PreparedStatement; import java.sql.Statement; import java.sql.Timestamp; import java.time.Instant; -import java.time.LocalDate; import java.util.List; import java.util.Optional; @@ -26,7 +25,7 @@ public class MemoryJdbcService { this.jdbcTemplate = jdbcTemplate; } - private static final RowMapper MEMORY_ROW_MAPPER = (rs, rowNum) -> new Memory( + private static final RowMapper MEMORY_ROW_MAPPER = (rs, _) -> new Memory( rs.getLong("id"), rs.getString("title"), rs.getString("description"), @@ -122,14 +121,6 @@ public class MemoryJdbcService { return jdbcTemplate.query(sql, MEMORY_ROW_MAPPER, user.getId()); } - public List findAllByUserAndYear(User user, int year) { - return jdbcTemplate.query( - "SELECT * FROM memory WHERE user_id = ? AND (extract(YEAR FROM start_date) = ? OR extract(YEAR FROM end_date) = ?) ORDER BY created_at DESC", - MEMORY_ROW_MAPPER, - user.getId(), year, year - ); - } - public List findAllByUserAndYear(User user, int year, String sortBy, String sortOrder) { String column = mapSortByToColumn(sortBy); String order = "desc".equalsIgnoreCase(sortOrder) ? "DESC" : "ASC"; @@ -137,19 +128,6 @@ public class MemoryJdbcService { return jdbcTemplate.query(sql, MEMORY_ROW_MAPPER, user.getId(), year, year); } - public List findByDateRange(User user, Instant startDate, Instant endDate) { - return jdbcTemplate.query( - "SELECT * FROM memory " + - "WHERE user_id = ? " + - "AND (end_date <= ? AND start_date >= ?) " + - "ORDER BY start_date DESC", - MEMORY_ROW_MAPPER, - user.getId(), - endDate, - startDate - ); - } - public List findDistinctYears(User user) { String sql = "SELECT DISTINCT EXTRACT(YEAR FROM start_date) " + "FROM memory " + diff --git a/src/main/java/com/dedicatedcode/reitti/repository/OptimisticLockException.java b/src/main/java/com/dedicatedcode/reitti/repository/OptimisticLockException.java index 121dca64..39fb7789 100644 --- a/src/main/java/com/dedicatedcode/reitti/repository/OptimisticLockException.java +++ b/src/main/java/com/dedicatedcode/reitti/repository/OptimisticLockException.java @@ -1,6 +1,6 @@ package com.dedicatedcode.reitti.repository; -public class OptimisticLockException extends Exception { +public class OptimisticLockException extends RuntimeException { public OptimisticLockException(String message) { super(message); } diff --git a/src/main/java/com/dedicatedcode/reitti/repository/PreviewRawLocationPointJdbcService.java b/src/main/java/com/dedicatedcode/reitti/repository/PreviewRawLocationPointJdbcService.java index 917d78f7..0616bca3 100644 --- a/src/main/java/com/dedicatedcode/reitti/repository/PreviewRawLocationPointJdbcService.java +++ b/src/main/java/com/dedicatedcode/reitti/repository/PreviewRawLocationPointJdbcService.java @@ -27,9 +27,11 @@ public class PreviewRawLocationPointJdbcService { rs.getLong("id"), rs.getTimestamp("timestamp").toInstant(), pointReaderWriter.read(rs.getString("geom")), - rs.getObject("elevation_meters", Double.class), rs.getDouble("accuracy_meters"), + rs.getObject("elevation_meters", Double.class), rs.getBoolean("processed"), + rs.getBoolean("synthetic"), + rs.getBoolean("ignored"), rs.getLong("version") ); @@ -38,7 +40,7 @@ public class PreviewRawLocationPointJdbcService { public List findByUserAndTimestampBetweenOrderByTimestampAsc( User user, String previewId, Instant startTime, Instant endTime) { - String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.version " + + String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version " + "FROM preview_raw_location_points rlp " + "WHERE rlp.user_id = ? AND rlp.timestamp BETWEEN ? AND ? AND preview_id = ? " + "ORDER BY rlp.timestamp"; @@ -47,7 +49,7 @@ public class PreviewRawLocationPointJdbcService { } public List findByUserAndProcessedIsFalseOrderByTimestampWithLimit(User user, String previewId, int limit, int offset) { - String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.version " + + String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version " + "FROM preview_raw_location_points rlp " + "WHERE rlp.user_id = ? AND rlp.processed = false AND preview_id = ? " + "ORDER BY rlp.timestamp " + @@ -57,12 +59,12 @@ public class PreviewRawLocationPointJdbcService { public List findClusteredPointsInTimeRangeForUser( User user, String previewId, Instant startTime, Instant endTime, int minimumPoints, double distanceInMeters) { - String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.version , " + + String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version , " + "ST_ClusterDBSCAN(rlp.geom, ?, ?) over () AS cluster_id " + "FROM preview_raw_location_points rlp " + "WHERE rlp.user_id = ? AND rlp.timestamp BETWEEN ? AND ? AND preview_id = ?"; - return jdbcTemplate.query(sql, (rs, rowNum) -> { + return jdbcTemplate.query(sql, (rs, _) -> { RawLocationPoint point = new RawLocationPoint( rs.getLong("id"), @@ -71,6 +73,8 @@ public class PreviewRawLocationPointJdbcService { rs.getDouble("accuracy_meters"), rs.getObject("elevation_meters", Double.class), rs.getBoolean("processed"), + rs.getBoolean("synthetic"), + rs.getBoolean("ignored"), rs.getLong("version") ); diff --git a/src/main/java/com/dedicatedcode/reitti/repository/PreviewTripJdbcService.java b/src/main/java/com/dedicatedcode/reitti/repository/PreviewTripJdbcService.java index b654cd89..8192bdea 100644 --- a/src/main/java/com/dedicatedcode/reitti/repository/PreviewTripJdbcService.java +++ b/src/main/java/com/dedicatedcode/reitti/repository/PreviewTripJdbcService.java @@ -14,7 +14,6 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.time.Instant; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; @Service @@ -64,9 +63,9 @@ public class PreviewTripJdbcService { Timestamp.from(startTime), Timestamp.from(endTime)); } - public void bulkInsert(User user, String previewId, List tripsToInsert) { + public List bulkInsert(User user, String previewId, List tripsToInsert) { if (tripsToInsert.isEmpty()) { - return; + return tripsToInsert; } String sql = """ @@ -92,6 +91,21 @@ public class PreviewTripJdbcService { .collect(Collectors.toList()); jdbcTemplate.batchUpdate(sql, batchArgs); + return tripsToInsert; } + public void deleteAll(List existingTrips) { + if (existingTrips == null || existingTrips.isEmpty()) { + return; + } + + List ids = existingTrips.stream() + .map(Trip::getId) + .toList(); + + String placeholders = String.join(",", ids.stream().map(id -> "?").toList()); + String sql = "DELETE FROM preview_trips WHERE id IN (" + placeholders + ")"; + + jdbcTemplate.update(sql, ids.toArray()); + } } diff --git a/src/main/java/com/dedicatedcode/reitti/repository/PreviewVisitDetectionParametersJdbcService.java b/src/main/java/com/dedicatedcode/reitti/repository/PreviewVisitDetectionParametersJdbcService.java index cdb124aa..25d89914 100644 --- a/src/main/java/com/dedicatedcode/reitti/repository/PreviewVisitDetectionParametersJdbcService.java +++ b/src/main/java/com/dedicatedcode/reitti/repository/PreviewVisitDetectionParametersJdbcService.java @@ -28,8 +28,6 @@ public class PreviewVisitDetectionParametersJdbcService { Instant validSince = validSinceTimestamp != null ? validSinceTimestamp.toInstant() : null; DetectionParameter.VisitDetection visitDetection = new DetectionParameter.VisitDetection( - rs.getLong("detection_search_distance_meters"), - rs.getInt("detection_minimum_adjacent_points"), rs.getLong("detection_minimum_stay_time_seconds"), rs.getLong("detection_max_merge_time_between_same_stay_points") ); @@ -40,7 +38,12 @@ public class PreviewVisitDetectionParametersJdbcService { rs.getLong("merging_min_distance_between_visits") ); - return new DetectionParameter(id, visitDetection, visitMerging, validSince, RecalculationState.DONE); + DetectionParameter.LocationDensity locationDensity = new DetectionParameter.LocationDensity( + rs.getDouble("density_max_interpolation_distance_meters"), + rs.getInt("density_max_interpolation_gap_minutes") + ); + + return new DetectionParameter(id, visitDetection, visitMerging, locationDensity, validSince, RecalculationState.DONE); }; public DetectionParameter findCurrent(User user, String previewId) { diff --git a/src/main/java/com/dedicatedcode/reitti/repository/PreviewVisitJdbcService.java b/src/main/java/com/dedicatedcode/reitti/repository/PreviewVisitJdbcService.java index b425b885..4a6d7587 100644 --- a/src/main/java/com/dedicatedcode/reitti/repository/PreviewVisitJdbcService.java +++ b/src/main/java/com/dedicatedcode/reitti/repository/PreviewVisitJdbcService.java @@ -70,7 +70,7 @@ public class PreviewVisitJdbcService { List createdVisits = new ArrayList<>(); String sql = """ INSERT INTO preview_visits (user_id, latitude, longitude, start_time, end_time, duration_seconds, processed, version, preview_id, preview_created_at) - VALUES (?, ?, ?, ?, ?, ?, false, 1, ?, now()); + VALUES (?, ?, ?, ?, ?, ?, false, 1, ?, now()) ON CONFLICT DO NOTHING; """; List batchArgs = visitsToInsert.stream() diff --git a/src/main/java/com/dedicatedcode/reitti/repository/RawLocationPointJdbcService.java b/src/main/java/com/dedicatedcode/reitti/repository/RawLocationPointJdbcService.java index 8a8d447a..a3972967 100644 --- a/src/main/java/com/dedicatedcode/reitti/repository/RawLocationPointJdbcService.java +++ b/src/main/java/com/dedicatedcode/reitti/repository/RawLocationPointJdbcService.java @@ -12,6 +12,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import java.sql.Timestamp; +import java.time.Duration; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZonedDateTime; @@ -39,6 +40,8 @@ public class RawLocationPointJdbcService { rs.getDouble("accuracy_meters"), rs.getObject("elevation_meters", Double.class), rs.getBoolean("processed"), + rs.getBoolean("synthetic"), + rs.getBoolean("ignored"), rs.getLong("version") ); @@ -49,7 +52,7 @@ public class RawLocationPointJdbcService { public List findByUserAndTimestampBetweenOrderByTimestampAsc( User user, Instant startTime, Instant endTime) { - String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.version " + + String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version " + "FROM raw_location_points rlp " + "WHERE rlp.user_id = ? AND rlp.timestamp BETWEEN ? AND ? " + "ORDER BY rlp.timestamp"; @@ -58,7 +61,7 @@ public class RawLocationPointJdbcService { } public List findByUserAndDateRange(User user, LocalDateTime startTime, LocalDateTime endTime) { - String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.version " + + String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version " + "FROM raw_location_points rlp " + "WHERE rlp.user_id = ? AND rlp.timestamp BETWEEN ? AND ? " + "ORDER BY rlp.timestamp"; @@ -67,7 +70,7 @@ public class RawLocationPointJdbcService { } public List findByUserAndProcessedIsFalseOrderByTimestamp(User user) { - String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.version " + + String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version " + "FROM raw_location_points rlp " + "WHERE rlp.user_id = ? AND rlp.processed = false " + "ORDER BY rlp.timestamp"; @@ -75,7 +78,7 @@ public class RawLocationPointJdbcService { } public List findByUserAndProcessedIsFalseOrderByTimestampWithLimit(User user, int limit, int offset) { - String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.version " + + String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version " + "FROM raw_location_points rlp " + "WHERE rlp.user_id = ? AND rlp.processed = false " + "ORDER BY rlp.timestamp " + @@ -92,34 +95,38 @@ public class RawLocationPointJdbcService { } public RawLocationPoint create(User user, RawLocationPoint rawLocationPoint) { - String sql = "INSERT INTO raw_location_points (user_id, timestamp, accuracy_meters, elevation_meters, geom, processed) " + - "VALUES (?, ?, ?, ?, ST_GeomFromText(?, '4326'), ?) RETURNING id"; + String sql = "INSERT INTO raw_location_points (user_id, timestamp, accuracy_meters, elevation_meters, geom, processed, synthetic, ignored) " + + "VALUES (?, ?, ?, ?, ST_GeomFromText(?, '4326'), ?, ?, ?) RETURNING id"; Long id = jdbcTemplate.queryForObject(sql, Long.class, user.getId(), Timestamp.from(rawLocationPoint.getTimestamp()), rawLocationPoint.getAccuracyMeters(), rawLocationPoint.getElevationMeters(), pointReaderWriter.write(rawLocationPoint.getGeom()), - rawLocationPoint.isProcessed() + rawLocationPoint.isProcessed(), + rawLocationPoint.isSynthetic(), + rawLocationPoint.isIgnored() ); return rawLocationPoint.withId(id); } public RawLocationPoint update(RawLocationPoint rawLocationPoint) { - String sql = "UPDATE raw_location_points SET timestamp = ?, accuracy_meters = ?, elevation_meters = ?, geom = ST_GeomFromText(?, '4326'), processed = ? WHERE id = ?"; + String sql = "UPDATE raw_location_points SET timestamp = ?, accuracy_meters = ?, elevation_meters = ?, geom = ST_GeomFromText(?, '4326'), processed = ?, synthetic = ?, ignored = ? WHERE id = ?"; jdbcTemplate.update(sql, Timestamp.from(rawLocationPoint.getTimestamp()), rawLocationPoint.getAccuracyMeters(), rawLocationPoint.getElevationMeters(), pointReaderWriter.write(rawLocationPoint.getGeom()), rawLocationPoint.isProcessed(), + rawLocationPoint.isSynthetic(), + rawLocationPoint.isIgnored(), rawLocationPoint.getId() ); return rawLocationPoint; } public Optional findById(Long id) { - String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.version " + + String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version " + "FROM raw_location_points rlp " + "WHERE rlp.id = ?"; List results = jdbcTemplate.query(sql, rawLocationPointRowMapper, id); @@ -127,7 +134,7 @@ public class RawLocationPointJdbcService { } public Optional findLatest(User user, Instant since) { - String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.version " + + String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version " + "FROM raw_location_points rlp " + "WHERE rlp.user_id = ? AND rlp.timestamp >= ? " + "ORDER BY rlp.timestamp LIMIT 1"; @@ -136,7 +143,7 @@ public class RawLocationPointJdbcService { } public Optional findLatest(User user) { - String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.version " + + String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version " + "FROM raw_location_points rlp " + "WHERE rlp.user_id = ? " + "ORDER BY rlp.timestamp DESC LIMIT 1"; @@ -151,7 +158,7 @@ public class RawLocationPointJdbcService { public List findClusteredPointsInTimeRangeForUser( User user, Instant startTime, Instant endTime, int minimumPoints, double distanceInMeters) { - String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.version , " + + String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version , " + "ST_ClusterDBSCAN(rlp.geom, ?, ?) over () AS cluster_id " + "FROM raw_location_points rlp " + "WHERE rlp.user_id = ? AND rlp.timestamp BETWEEN ? AND ?"; @@ -165,6 +172,8 @@ public class RawLocationPointJdbcService { rs.getDouble("accuracy_meters"), rs.getObject("elevation_meters", Double.class), rs.getBoolean("processed"), + rs.getBoolean("synthetic"), + rs.getBoolean("ignored"), rs.getLong("version") ); @@ -198,6 +207,8 @@ public class RawLocationPointJdbcService { accuracy_meters, elevation_meters, processed, + ignored, + synthetic, version, ST_Within(geom, ST_MakeEnvelope(?, ?, ?, ?, 4326)) as in_box, LAG(ST_Within(geom, ST_MakeEnvelope(?, ?, ?, ?, 4326))) @@ -207,6 +218,7 @@ public class RawLocationPointJdbcService { FROM raw_location_points WHERE user_id = ? AND timestamp BETWEEN ? AND ? + AND ignored = false ) SELECT id, @@ -216,6 +228,8 @@ public class RawLocationPointJdbcService { accuracy_meters, elevation_meters, processed, + synthetic, + ignored, version FROM all_points WHERE in_box = true @@ -249,8 +263,8 @@ public class RawLocationPointJdbcService { return -1; } - String sql = "INSERT INTO raw_location_points (user_id, timestamp, accuracy_meters, elevation_meters, geom, processed) " + - "VALUES (?, ?, ?, ?, CAST(? AS geometry), false) ON CONFLICT DO NOTHING;"; + String sql = "INSERT INTO raw_location_points (user_id, timestamp, accuracy_meters, elevation_meters, geom, processed, synthetic, ignored) " + + "VALUES (?, ?, ?, ?, CAST(? AS geometry), false, false, false) ON CONFLICT DO NOTHING;"; List batchArgs = new ArrayList<>(); for (LocationPoint point : points) { @@ -328,4 +342,140 @@ public class RawLocationPointJdbcService { Timestamp.from(start)); return count != null && count > 0; } + + // New methods for density normalization + + public List findSurroundingPoints(User user, Instant timestamp, Duration window) { + Instant start = timestamp.minus(window); + Instant end = timestamp.plus(window); + + String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version " + + "FROM raw_location_points rlp " + + "WHERE rlp.user_id = ? AND rlp.timestamp BETWEEN ? AND ? " + + "ORDER BY rlp.timestamp"; + return jdbcTemplate.query(sql, rawLocationPointRowMapper, + user.getId(), Timestamp.from(start), Timestamp.from(end)); + } + + public List findSyntheticPointsInRange(User user, Instant start, Instant end) { + String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version " + + "FROM raw_location_points rlp " + + "WHERE rlp.user_id = ? AND rlp.timestamp BETWEEN ? AND ? AND rlp.synthetic = true " + + "ORDER BY rlp.timestamp"; + return jdbcTemplate.query(sql, rawLocationPointRowMapper, + user.getId(), Timestamp.from(start), Timestamp.from(end)); + } + + public int bulkInsertSynthetic(User user, List syntheticPoints) { + if (syntheticPoints.isEmpty()) { + return 0; + } + + String sql = "INSERT INTO raw_location_points (user_id, timestamp, accuracy_meters, elevation_meters, geom, processed, synthetic, ignored) " + + "VALUES (?, ?, ?, ?, CAST(? AS geometry), false, true, false) ON CONFLICT DO NOTHING;"; + + List batchArgs = new ArrayList<>(); + for (LocationPoint point : syntheticPoints) { + ZonedDateTime parse = ZonedDateTime.parse(point.getTimestamp()); + Timestamp timestamp = Timestamp.from(parse.toInstant()); + batchArgs.add(new Object[]{ + user.getId(), + timestamp, + point.getAccuracyMeters(), + point.getElevationMeters(), + geometryFactory.createPoint(new Coordinate(point.getLongitude(), point.getLatitude())).toString() + }); + } + int[] ints = jdbcTemplate.batchUpdate(sql, batchArgs); + return Arrays.stream(ints).sum(); + } + + public void deleteSyntheticPointsInRange(User user, Instant start, Instant end) { + String sql = "DELETE FROM raw_location_points WHERE user_id = ? AND timestamp BETWEEN ? AND ? AND synthetic = true"; + jdbcTemplate.update(sql, user.getId(), Timestamp.from(start), Timestamp.from(end)); + } + + public void bulkUpdateIgnoredStatus(List pointIds, boolean ignored) { + if (pointIds.isEmpty()) { + return; + } + + String sql = "UPDATE raw_location_points SET ignored = ? WHERE id = ?"; + + List batchArgs = pointIds.stream() + .map(pointId -> new Object[]{ignored, pointId}) + .collect(Collectors.toList()); + + jdbcTemplate.batchUpdate(sql, batchArgs); + } + + public List findByUserAndTimeRangeWithFlags(User user, Instant start, Instant end, Boolean synthetic, Boolean ignored) { + StringBuilder sqlBuilder = new StringBuilder(); + sqlBuilder.append("SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version "); + sqlBuilder.append("FROM raw_location_points rlp "); + sqlBuilder.append("WHERE rlp.user_id = ? AND rlp.timestamp BETWEEN ? AND ? "); + + List params = new ArrayList<>(); + params.add(user.getId()); + params.add(Timestamp.from(start)); + params.add(Timestamp.from(end)); + + if (synthetic != null) { + sqlBuilder.append("AND rlp.synthetic = ? "); + params.add(synthetic); + } + + if (ignored != null) { + sqlBuilder.append("AND rlp.ignored = ? "); + params.add(ignored); + } + + sqlBuilder.append("ORDER BY rlp.timestamp"); + + return jdbcTemplate.query(sqlBuilder.toString(), rawLocationPointRowMapper, params.toArray()); + } + + public int bulkUpsertSynthetic(User user, List toInsert) { + if (toInsert.isEmpty()) { + return 0; + } + + String sql = """ + INSERT INTO raw_location_points (user_id, timestamp, accuracy_meters, elevation_meters, geom, processed, synthetic, ignored) + VALUES (?, ?, ?, ?, CAST(? AS geometry), false, true, false) + ON CONFLICT (user_id, timestamp) DO UPDATE SET + accuracy_meters = EXCLUDED.accuracy_meters, + elevation_meters = EXCLUDED.elevation_meters, + geom = EXCLUDED.geom, + processed = false, + synthetic = true, + ignored = false; + """; + + List batchArgs = new ArrayList<>(); + for (LocationPoint point : toInsert) { + ZonedDateTime parse = ZonedDateTime.parse(point.getTimestamp()); + Timestamp timestamp = Timestamp.from(parse.toInstant()); + batchArgs.add(new Object[]{ + user.getId(), + timestamp, + point.getAccuracyMeters(), + point.getElevationMeters(), + geometryFactory.createPoint(new Coordinate(point.getLongitude(), point.getLatitude())).toString() + }); + } + int[] ints = jdbcTemplate.batchUpdate(sql, batchArgs); + return Arrays.stream(ints).sum(); + } + + public void deleteSyntheticByIds(List toDelete) { + if (toDelete == null || toDelete.isEmpty()) { + return; + } + + String placeholders = String.join(",", toDelete.stream().map(id -> "?").toList()); + String sql = "DELETE FROM raw_location_points WHERE id IN (" + placeholders + ") AND synthetic = true"; + + jdbcTemplate.update(sql, toDelete.toArray()); + } } diff --git a/src/main/java/com/dedicatedcode/reitti/repository/SignificantPlaceOverrideJdbcService.java b/src/main/java/com/dedicatedcode/reitti/repository/SignificantPlaceOverrideJdbcService.java index b8cd6c27..c17e2045 100644 --- a/src/main/java/com/dedicatedcode/reitti/repository/SignificantPlaceOverrideJdbcService.java +++ b/src/main/java/com/dedicatedcode/reitti/repository/SignificantPlaceOverrideJdbcService.java @@ -22,7 +22,7 @@ public class SignificantPlaceOverrideJdbcService { } public Optional findByUserAndPoint(User user, GeoPoint point) { - double meterInDegrees = GeoUtils.metersToDegreesAtPosition(5.0, point.latitude())[0]; + double meterInDegrees = GeoUtils.metersToDegreesAtPosition(5.0, point.latitude()); String sql = "SELECT name, category, timezone FROM significant_places_overrides WHERE user_id = ? AND ST_DWithin(geom, ST_GeomFromText(?, '4326'), ?) ORDER BY ST_Distance(geom, ST_GeomFromText(?, '4326')) ASC LIMIT 1"; List override = jdbcTemplate.query(sql, (rs, rowNum) -> new PlaceInformationOverride( rs.getString("name"), @@ -38,7 +38,7 @@ public class SignificantPlaceOverrideJdbcService { public void insertOverride(User user, SignificantPlace place) { GeoPoint point = new GeoPoint(place.getLatitudeCentroid(), place.getLongitudeCentroid()); - double meterInDegrees = GeoUtils.metersToDegreesAtPosition(5.0, place.getLatitudeCentroid())[0]; + double meterInDegrees = GeoUtils.metersToDegreesAtPosition(5.0, place.getLatitudeCentroid()); this.jdbcTemplate.update("DELETE FROM significant_places_overrides WHERE user_id = ? AND ST_DWithin(geom, ST_GeomFromText(?, '4326'), ?)", user.getId(), pointReaderWriter.write(point), meterInDegrees); String sql = "INSERT INTO significant_places_overrides (user_id, geom, name, category, timezone) VALUES (?, ST_GeomFromText(?, '4326'), ?, ?, ?)"; jdbcTemplate.update(sql, user.getId(), pointReaderWriter.write(point), place.getName(), place.getType().name(), place.getTimezone().getId()); diff --git a/src/main/java/com/dedicatedcode/reitti/repository/TripJdbcService.java b/src/main/java/com/dedicatedcode/reitti/repository/TripJdbcService.java index 4a09b35d..95a7b8bf 100644 --- a/src/main/java/com/dedicatedcode/reitti/repository/TripJdbcService.java +++ b/src/main/java/com/dedicatedcode/reitti/repository/TripJdbcService.java @@ -13,7 +13,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; import java.time.Instant; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -49,20 +48,6 @@ public class TripJdbcService { } }; - public List findByIds(User user, List ids) { - if (ids == null || ids.isEmpty()) { - return List.of(); - } - String placeholders = String.join(",", Collections.nCopies(ids.size(), "?")); - String sql = "SELECT t.* FROM trips t WHERE t.user_id = ? AND t.id IN (" + placeholders + ")"; - Object[] params = new Object[ids.size() + 1]; - params[0] = user.getId(); - for (int i = 0; i < ids.size(); i++) { - params[i + 1] = ids.get(i); - } - return jdbcTemplate.query(sql, TRIP_ROW_MAPPER, params); - } - public List findByUser(User user) { String sql = "SELECT t.*" + "FROM trips t " + @@ -166,9 +151,9 @@ public class TripJdbcService { return results.isEmpty() ? Optional.empty() : Optional.of(results.getFirst()); } - public void bulkInsert(User user, List tripsToInsert) { + public List bulkInsert(User user, List tripsToInsert) { if (tripsToInsert.isEmpty()) { - return; + return tripsToInsert; } String sql = """ @@ -193,6 +178,7 @@ public class TripJdbcService { .collect(Collectors.toList()); jdbcTemplate.batchUpdate(sql, batchArgs); + return tripsToInsert; } public void deleteAll() { @@ -205,16 +191,27 @@ public class TripJdbcService { jdbcTemplate.update(sql, user.getId()); } - public void deleteAllForUserBetween(User user, Instant start, Instant end) { - String sql = "DELETE FROM trips WHERE user_id = ? AND start_time <= ? AND end_time >= ?"; - jdbcTemplate.update(sql, user.getId(), Timestamp.from(end), Timestamp.from(start)); - } - public void deleteAllForUserAfter(User user, Instant start) { - String sql = "DELETE FROM trips WHERE user_id = ? AND end_time >= ?"; - jdbcTemplate.update(sql, user.getId(), Timestamp.from(start)); - } - public List findIdsByUser(User user) { return jdbcTemplate.queryForList("SELECT id FROM trips WHERE user_id = ?", Long.class, user.getId()); } + + @SuppressWarnings("DataFlowIssue") + public long count() { + return jdbcTemplate.queryForObject("SELECT COUNT(*) FROM trips", Long.class); + } + + public void deleteAll(List existingTrips) { + if (existingTrips == null || existingTrips.isEmpty()) { + return; + } + + List ids = existingTrips.stream() + .map(Trip::getId) + .toList(); + + String placeholders = String.join(",", ids.stream().map(id -> "?").toList()); + String sql = "DELETE FROM trips WHERE id IN (" + placeholders + ")"; + + jdbcTemplate.update(sql, ids.toArray()); + } } diff --git a/src/main/java/com/dedicatedcode/reitti/repository/VisitDetectionParametersJdbcService.java b/src/main/java/com/dedicatedcode/reitti/repository/VisitDetectionParametersJdbcService.java index a04accf0..ce547407 100644 --- a/src/main/java/com/dedicatedcode/reitti/repository/VisitDetectionParametersJdbcService.java +++ b/src/main/java/com/dedicatedcode/reitti/repository/VisitDetectionParametersJdbcService.java @@ -32,8 +32,6 @@ public class VisitDetectionParametersJdbcService { Instant validSince = validSinceTimestamp != null ? validSinceTimestamp.toInstant() : null; DetectionParameter.VisitDetection visitDetection = new DetectionParameter.VisitDetection( - rs.getLong("detection_search_distance_meters"), - rs.getInt("detection_minimum_adjacent_points"), rs.getLong("detection_minimum_stay_time_seconds"), rs.getLong("detection_max_merge_time_between_same_stay_points") ); @@ -44,7 +42,12 @@ public class VisitDetectionParametersJdbcService { rs.getLong("merging_min_distance_between_visits") ); - return new DetectionParameter(id, visitDetection, visitMerging, validSince, recalculationState); + DetectionParameter.LocationDensity locationDensity = new DetectionParameter.LocationDensity( + rs.getDouble("density_max_interpolation_distance_meters"), + rs.getInt("density_max_interpolation_gap_minutes") + ); + + return new DetectionParameter(id, visitDetection, visitMerging, locationDensity, validSince, recalculationState); }; @@ -76,10 +79,10 @@ public class VisitDetectionParametersJdbcService { public void saveConfiguration(User user, DetectionParameter detectionParameter) { String sql = """ INSERT INTO visit_detection_parameters ( - user_id, valid_since, detection_search_distance_meters, - detection_minimum_adjacent_points, detection_minimum_stay_time_seconds, + user_id, valid_since, detection_minimum_stay_time_seconds, detection_max_merge_time_between_same_stay_points, merging_search_duration_in_hours, - merging_max_merge_time_between_same_visits, merging_min_distance_between_visits, recalculation_state) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + merging_max_merge_time_between_same_visits, merging_min_distance_between_visits, + density_max_interpolation_distance_meters, density_max_interpolation_gap_minutes, recalculation_state) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """; Timestamp validSinceTimestamp = detectionParameter.getValidSince() != null ? @@ -88,13 +91,13 @@ public class VisitDetectionParametersJdbcService { jdbcTemplate.update(sql, user.getId(), validSinceTimestamp, - detectionParameter.getVisitDetection().getSearchDistanceInMeters(), - detectionParameter.getVisitDetection().getMinimumAdjacentPoints(), detectionParameter.getVisitDetection().getMinimumStayTimeInSeconds(), detectionParameter.getVisitDetection().getMaxMergeTimeBetweenSameStayPoints(), detectionParameter.getVisitMerging().getSearchDurationInHours(), detectionParameter.getVisitMerging().getMaxMergeTimeBetweenSameVisits(), detectionParameter.getVisitMerging().getMinDistanceBetweenVisits(), + detectionParameter.getLocationDensity().getMaxInterpolationDistanceMeters(), + detectionParameter.getLocationDensity().getMaxInterpolationGapMinutes(), detectionParameter.getRecalculationState().name() ); } @@ -104,13 +107,13 @@ public class VisitDetectionParametersJdbcService { String sql = """ UPDATE visit_detection_parameters SET valid_since = ?, - detection_search_distance_meters = ?, - detection_minimum_adjacent_points = ?, detection_minimum_stay_time_seconds = ?, detection_max_merge_time_between_same_stay_points = ?, merging_search_duration_in_hours = ?, merging_max_merge_time_between_same_visits = ?, merging_min_distance_between_visits = ?, + density_max_interpolation_distance_meters = ?, + density_max_interpolation_gap_minutes = ?, recalculation_state = ? WHERE id = ? """; @@ -120,13 +123,13 @@ public class VisitDetectionParametersJdbcService { jdbcTemplate.update(sql, validSinceTimestamp, - detectionParameter.getVisitDetection().getSearchDistanceInMeters(), - detectionParameter.getVisitDetection().getMinimumAdjacentPoints(), detectionParameter.getVisitDetection().getMinimumStayTimeInSeconds(), detectionParameter.getVisitDetection().getMaxMergeTimeBetweenSameStayPoints(), detectionParameter.getVisitMerging().getSearchDurationInHours(), detectionParameter.getVisitMerging().getMaxMergeTimeBetweenSameVisits(), detectionParameter.getVisitMerging().getMinDistanceBetweenVisits(), + detectionParameter.getLocationDensity().getMaxInterpolationDistanceMeters(), + detectionParameter.getLocationDensity().getMaxInterpolationGapMinutes(), detectionParameter.getRecalculationState().name(), detectionParameter.getId() ); diff --git a/src/main/java/com/dedicatedcode/reitti/repository/VisitJdbcService.java b/src/main/java/com/dedicatedcode/reitti/repository/VisitJdbcService.java index 753d137a..e8234ea9 100644 --- a/src/main/java/com/dedicatedcode/reitti/repository/VisitJdbcService.java +++ b/src/main/java/com/dedicatedcode/reitti/repository/VisitJdbcService.java @@ -136,7 +136,16 @@ public class VisitJdbcService { List createdVisits = new ArrayList<>(); String sql = """ INSERT INTO visits (user_id, latitude, longitude, start_time, end_time, duration_seconds, processed, version) - VALUES (?, ?, ?, ?, ?, ?, false, 1) ON CONFLICT DO NOTHING; + VALUES (?, ?, ?, ?, ?, ?, false, 1) ON CONFLICT (user_id, start_time, end_time) DO UPDATE SET + user_id = EXCLUDED.user_id, + latitude = EXCLUDED.latitude, + longitude = EXCLUDED.longitude, + start_time = EXCLUDED.start_time, + end_time = EXCLUDED.end_time, + duration_seconds = EXCLUDED.duration_seconds, + processed = EXCLUDED.processed, + id = visits.id, + version = visits.version + 1; """; List batchArgs = visitsToInsert.stream() @@ -189,4 +198,8 @@ public class VisitJdbcService { public void deleteAllForUserAfter(User user, Instant start) { jdbcTemplate.update("DELETE FROM visits WHERE user_id = ? AND end_time >= ?", user.getId(), Timestamp.from(start)); } + + public long count() { + return jdbcTemplate.queryForObject("SELECT COUNT(*) FROM visits", Long.class); + } } diff --git a/src/main/java/com/dedicatedcode/reitti/service/ImportBatchProcessor.java b/src/main/java/com/dedicatedcode/reitti/service/ImportBatchProcessor.java index 5497c569..38ac7c0c 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/ImportBatchProcessor.java +++ b/src/main/java/com/dedicatedcode/reitti/service/ImportBatchProcessor.java @@ -1,55 +1,58 @@ package com.dedicatedcode.reitti.service; -import com.dedicatedcode.reitti.config.RabbitMQConfig; import com.dedicatedcode.reitti.dto.LocationPoint; import com.dedicatedcode.reitti.event.LocationDataEvent; import com.dedicatedcode.reitti.event.TriggerProcessingEvent; import com.dedicatedcode.reitti.model.security.User; +import com.dedicatedcode.reitti.service.processing.LocationDataIngestPipeline; +import com.dedicatedcode.reitti.service.processing.ProcessingPipelineTrigger; import jakarta.annotation.PreDestroy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.List; +import java.util.UUID; import java.util.concurrent.*; @Component public class ImportBatchProcessor { private static final Logger logger = LoggerFactory.getLogger(ImportBatchProcessor.class); - - private final RabbitTemplate rabbitTemplate; + + private final LocationDataIngestPipeline locationDataIngestPipeline; private final int batchSize; private final int processingIdleStartTime; + private final ProcessingPipelineTrigger processingPipelineTrigger; private final ScheduledExecutorService scheduler; private final ConcurrentHashMap> pendingTriggers; public ImportBatchProcessor( - RabbitTemplate rabbitTemplate, + LocationDataIngestPipeline locationDataIngestPipeline, @Value("${reitti.import.batch-size:100}") int batchSize, - @Value("${reitti.import.processing-idle-start-time:15}") int processingIdleStartTime) { - this.rabbitTemplate = rabbitTemplate; + @Value("${reitti.import.processing-idle-start-time:15}") int processingIdleStartTime, + ProcessingPipelineTrigger processingPipelineTrigger) { + this.locationDataIngestPipeline = locationDataIngestPipeline; this.batchSize = batchSize; this.processingIdleStartTime = processingIdleStartTime; + this.processingPipelineTrigger = processingPipelineTrigger; this.scheduler = Executors.newScheduledThreadPool(2); this.pendingTriggers = new ConcurrentHashMap<>(); } - public void sendToQueue(User user, List batch) { + public void processBatch(User user, List batch) { LocationDataEvent event = new LocationDataEvent( user.getUsername(), - new ArrayList<>(batch) + new ArrayList<>(batch), + UUID.randomUUID().toString() ); - rabbitTemplate.convertAndSend( - RabbitMQConfig.EXCHANGE_NAME, - RabbitMQConfig.LOCATION_DATA_ROUTING_KEY, - event - ); - logger.info("Queued batch of {} locations for processing", batch.size()); - + logger.debug("Sending batch of {} locations for storing", batch.size()); + locationDataIngestPipeline.processLocationData(event); + logger.debug("Sending batch of {} locations for processing", batch.size()); + TriggerProcessingEvent triggerEvent = new TriggerProcessingEvent(user.getUsername(), null, UUID.randomUUID().toString()); + processingPipelineTrigger.handle(triggerEvent); scheduleProcessingTrigger(user.getUsername()); } @@ -61,13 +64,10 @@ public class ImportBatchProcessor { ScheduledFuture newTrigger = scheduler.schedule(() -> { try { - TriggerProcessingEvent triggerEvent = new TriggerProcessingEvent(username, null); - rabbitTemplate.convertAndSend( - RabbitMQConfig.EXCHANGE_NAME, - RabbitMQConfig.TRIGGER_PROCESSING_PIPELINE_ROUTING_KEY, - triggerEvent - ); - logger.info("Triggered processing for user: {}", username); + logger.debug("Triggered processing for user: {}", username); + TriggerProcessingEvent triggerEvent = new TriggerProcessingEvent(username, null, UUID.randomUUID().toString()); + processingPipelineTrigger.handle(triggerEvent); + pendingTriggers.remove(username); } catch (Exception e) { logger.error("Failed to trigger processing for user: {}", username, e); diff --git a/src/main/java/com/dedicatedcode/reitti/service/MessageDispatcherService.java b/src/main/java/com/dedicatedcode/reitti/service/MessageDispatcherService.java index 70a2729b..2589520f 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/MessageDispatcherService.java +++ b/src/main/java/com/dedicatedcode/reitti/service/MessageDispatcherService.java @@ -1,10 +1,12 @@ package com.dedicatedcode.reitti.service; import com.dedicatedcode.reitti.config.RabbitMQConfig; -import com.dedicatedcode.reitti.event.*; +import com.dedicatedcode.reitti.event.SSEEvent; +import com.dedicatedcode.reitti.event.SignificantPlaceCreatedEvent; +import com.dedicatedcode.reitti.event.TriggerProcessingEvent; import com.dedicatedcode.reitti.repository.UserJdbcService; import com.dedicatedcode.reitti.service.geocoding.ReverseGeocodingListener; -import com.dedicatedcode.reitti.service.processing.*; +import com.dedicatedcode.reitti.service.processing.ProcessingPipelineTrigger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.amqp.rabbit.annotation.RabbitListener; @@ -16,10 +18,6 @@ public class MessageDispatcherService { private static final Logger logger = LoggerFactory.getLogger(MessageDispatcherService.class); - private final LocationDataIngestPipeline locationDataIngestPipeline; - private final VisitDetectionService visitDetectionService; - private final VisitMergingService visitMergingService; - private final TripDetectionService tripDetectionService; private final ReverseGeocodingListener reverseGeocodingListener; private final ProcessingPipelineTrigger processingPipelineTrigger; private final UserSseEmitterService userSseEmitterService; @@ -27,19 +25,11 @@ public class MessageDispatcherService { private final VisitDetectionPreviewService visitDetectionPreviewService; @Autowired - public MessageDispatcherService(LocationDataIngestPipeline locationDataIngestPipeline, - VisitDetectionService visitDetectionService, - VisitMergingService visitMergingService, - TripDetectionService tripDetectionService, - ReverseGeocodingListener reverseGeocodingListener, + public MessageDispatcherService(ReverseGeocodingListener reverseGeocodingListener, ProcessingPipelineTrigger processingPipelineTrigger, UserSseEmitterService userSseEmitterService, UserJdbcService userJdbcService, VisitDetectionPreviewService visitDetectionPreviewService) { - this.locationDataIngestPipeline = locationDataIngestPipeline; - this.visitDetectionService = visitDetectionService; - this.visitMergingService = visitMergingService; - this.tripDetectionService = tripDetectionService; this.reverseGeocodingListener = reverseGeocodingListener; this.processingPipelineTrigger = processingPipelineTrigger; this.userSseEmitterService = userSseEmitterService; @@ -47,55 +37,22 @@ public class MessageDispatcherService { this.visitDetectionPreviewService = visitDetectionPreviewService; } - @RabbitListener(queues = RabbitMQConfig.LOCATION_DATA_QUEUE, concurrency = "${reitti.events.concurrency}") - public void handleLocationData(LocationDataEvent event) { - logger.debug("Dispatching LocationDataEvent for user: {}", event.getUsername()); - locationDataIngestPipeline.processLocationData(event); - } - - @RabbitListener(queues = RabbitMQConfig.STAY_DETECTION_QUEUE, concurrency = "${reitti.events.concurrency}") - public void handleStayDetection(LocationProcessEvent event) { - logger.debug("Dispatching LocationProcessEvent for user: {}", event.getUsername()); - visitDetectionService.detectStayPoints(event); - visitDetectionPreviewService.updatePreviewStatus(event.getPreviewId()); - } - - @RabbitListener(queues = RabbitMQConfig.MERGE_VISIT_QUEUE, concurrency = "1") - public void handleVisitMerging(VisitUpdatedEvent event) { - logger.debug("Dispatching VisitUpdatedEvent for user: {}", event.getUsername()); - visitMergingService.visitUpdated(event); - visitDetectionPreviewService.updatePreviewStatus(event.getPreviewId()); - } - - @RabbitListener(queues = RabbitMQConfig.DETECT_TRIP_QUEUE, concurrency = "${reitti.events.concurrency}") - public void handleTripDetection(ProcessedVisitCreatedEvent event) { - logger.debug("Dispatching ProcessedVisitCreatedEvent for user: {}", event.getUsername()); - tripDetectionService.visitCreated(event); - visitDetectionPreviewService.updatePreviewStatus(event.getPreviewId()); - } - - @RabbitListener(queues = RabbitMQConfig.RECALCULATE_TRIP_QUEUE, concurrency = "${reitti.events.concurrency}") - public void handleTripDetection(RecalculateTripEvent event) { - logger.debug("Dispatching RecalculateTripEvent for user: {}", event.getUsername()); - tripDetectionService.recalculateTrip(event); - } - @RabbitListener(queues = RabbitMQConfig.SIGNIFICANT_PLACE_QUEUE, concurrency = "${reitti.events.concurrency}") public void handleSignificantPlaceCreated(SignificantPlaceCreatedEvent event) { - logger.debug("Dispatching SignificantPlaceCreatedEvent for place: {}", event.placeId()); + logger.info("Dispatching SignificantPlaceCreatedEvent: {}", event); reverseGeocodingListener.handleSignificantPlaceCreated(event); visitDetectionPreviewService.updatePreviewStatus(event.previewId()); } @RabbitListener(queues = RabbitMQConfig.USER_EVENT_QUEUE) public void handleUserNotificationEvent(SSEEvent event) { - logger.debug("Dispatching SSEEvent for user: {}", event.getUserId()); + logger.debug("Dispatching SSEEvent: {}", event); this.userJdbcService.findById(event.getUserId()).ifPresentOrElse(user -> this.userSseEmitterService.sendEventToUser(user, event), () -> logger.warn("User not found for user: {}", event.getUserId())); } @RabbitListener(queues = RabbitMQConfig.TRIGGER_PROCESSING_PIPELINE_QUEUE, concurrency = "${reitti.events.concurrency}") public void handleTriggerProcessingEvent(TriggerProcessingEvent event) { - logger.debug("Dispatching TriggerProcessingEvent for user: {}", event.getUsername()); + logger.info("Dispatching TriggerProcessingEvent {}", event); processingPipelineTrigger.handle(event); visitDetectionPreviewService.updatePreviewStatus(event.getPreviewId()); } diff --git a/src/main/java/com/dedicatedcode/reitti/service/QueueStatsService.java b/src/main/java/com/dedicatedcode/reitti/service/QueueStatsService.java index 8b465460..dcf8f5e1 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/QueueStatsService.java +++ b/src/main/java/com/dedicatedcode/reitti/service/QueueStatsService.java @@ -25,10 +25,7 @@ public class QueueStatsService { private final List QUEUES = List.of( RabbitMQConfig.LOCATION_DATA_QUEUE, - RabbitMQConfig.STAY_DETECTION_QUEUE, - RabbitMQConfig.MERGE_VISIT_QUEUE, RabbitMQConfig.SIGNIFICANT_PLACE_QUEUE, - RabbitMQConfig.DETECT_TRIP_QUEUE, RabbitMQConfig.USER_EVENT_QUEUE ); @@ -172,10 +169,7 @@ public class QueueStatsService { private String getMessageKeyForQueue(String queueName, String suffix) { return switch (queueName) { case RabbitMQConfig.LOCATION_DATA_QUEUE -> "queue.location.data." + suffix; - case RabbitMQConfig.STAY_DETECTION_QUEUE -> "queue.stay.detection." + suffix; - case RabbitMQConfig.MERGE_VISIT_QUEUE -> "queue.merge.visit." + suffix; case RabbitMQConfig.SIGNIFICANT_PLACE_QUEUE -> "queue.significant.place." + suffix; - case RabbitMQConfig.DETECT_TRIP_QUEUE -> "queue.detect.trip." + suffix; case RabbitMQConfig.USER_EVENT_QUEUE -> "queue.user.event." + suffix; default -> "queue.unknown." + suffix; }; diff --git a/src/main/java/com/dedicatedcode/reitti/service/StorageService.java b/src/main/java/com/dedicatedcode/reitti/service/StorageService.java index 722194cd..2115aced 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/StorageService.java +++ b/src/main/java/com/dedicatedcode/reitti/service/StorageService.java @@ -17,6 +17,13 @@ public class StorageService { public StorageService(@Value("${reitti.storage.path}") String storagePath) { this.storagePath = storagePath; Path path = Paths.get(storagePath); + try { + // Create directory if it doesn't exist + Files.createDirectories(path); + } catch (IOException e) { + throw new RuntimeException("Failed to create storage directory '" + storagePath + "': " + e.getMessage(), e); + } + if (!Files.isWritable(path)) { throw new RuntimeException("Storage path '" + storagePath + "' is not writable. Please ensure the directory exists and the application has write permissions."); } diff --git a/src/main/java/com/dedicatedcode/reitti/service/UserService.java b/src/main/java/com/dedicatedcode/reitti/service/UserService.java index 886addbb..521624a3 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/UserService.java +++ b/src/main/java/com/dedicatedcode/reitti/service/UserService.java @@ -118,8 +118,9 @@ public class UserService { private void saveDefaultVisitDetectionParameters(User createdUser) { visitDetectionParametersJdbcService.saveConfiguration(createdUser, new DetectionParameter(null, - new DetectionParameter.VisitDetection(100, 5, 300, 330), + new DetectionParameter.VisitDetection(300, 300), new DetectionParameter.VisitMerging(48, 300, 200), + new DetectionParameter.LocationDensity(50, 1440), null, RecalculationState.DONE) ); diff --git a/src/main/java/com/dedicatedcode/reitti/service/VisitDetectionPreviewService.java b/src/main/java/com/dedicatedcode/reitti/service/VisitDetectionPreviewService.java index eb03a07b..9784f7a5 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/VisitDetectionPreviewService.java +++ b/src/main/java/com/dedicatedcode/reitti/service/VisitDetectionPreviewService.java @@ -6,6 +6,7 @@ import com.dedicatedcode.reitti.model.processing.DetectionParameter; import com.dedicatedcode.reitti.model.security.User; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.amqp.rabbit.annotation.RabbitListener; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; @@ -39,13 +40,11 @@ public class VisitDetectionPreviewService { String previewId = UUID.randomUUID().toString(); this.jdbcTemplate.update(""" - INSERT INTO preview_visit_detection_parameters(user_id, valid_since, detection_search_distance_meters, detection_minimum_adjacent_points, detection_minimum_stay_time_seconds, + INSERT INTO preview_visit_detection_parameters(user_id, valid_since, detection_minimum_stay_time_seconds, detection_max_merge_time_between_same_stay_points, merging_search_duration_in_hours, merging_max_merge_time_between_same_visits, merging_min_distance_between_visits, preview_id, preview_created_at) - VALUES (?,?,?,?,?,?,?,?,?,?,?)""", + VALUES (?,?,?,?,?,?,?,?,?)""", user.getId(), config.getValidSince() != null ? Timestamp.from(config.getValidSince()) : null, - config.getVisitDetection().getSearchDistanceInMeters(), - config.getVisitDetection().getMinimumAdjacentPoints(), config.getVisitDetection().getMinimumStayTimeInSeconds(), config.getVisitDetection().getMaxMergeTimeBetweenSameStayPoints(), config.getVisitMerging().getSearchDurationInHours(), @@ -66,7 +65,7 @@ public class VisitDetectionPreviewService { user.getId()); log.debug("Copied preview data user [{}] with previewId [{}] successfully", user.getId(), previewId); - TriggerProcessingEvent triggerEvent = new TriggerProcessingEvent(user.getUsername(), previewId); + TriggerProcessingEvent triggerEvent = new TriggerProcessingEvent(user.getUsername(), previewId, UUID.randomUUID().toString()); rabbitTemplate.convertAndSend( RabbitMQConfig.EXCHANGE_NAME, RabbitMQConfig.TRIGGER_PROCESSING_PIPELINE_ROUTING_KEY, @@ -87,6 +86,7 @@ public class VisitDetectionPreviewService { return Instant.now().minusSeconds(READY_THRESHOLD_SECONDS).isAfter(lastUpdate); } + public void updatePreviewStatus(String previewId) { if (previewId != null) { log.debug("Updating preview status for previewId: {}", previewId); diff --git a/src/main/java/com/dedicatedcode/reitti/service/importer/BaseGoogleTimelineImporter.java b/src/main/java/com/dedicatedcode/reitti/service/importer/BaseGoogleTimelineImporter.java index a8aed6b6..9ec45a99 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/importer/BaseGoogleTimelineImporter.java +++ b/src/main/java/com/dedicatedcode/reitti/service/importer/BaseGoogleTimelineImporter.java @@ -1,15 +1,12 @@ package com.dedicatedcode.reitti.service.importer; import com.dedicatedcode.reitti.dto.LocationPoint; -import com.dedicatedcode.reitti.model.processing.DetectionParameter; import com.dedicatedcode.reitti.model.security.User; import com.dedicatedcode.reitti.service.ImportBatchProcessor; -import com.dedicatedcode.reitti.service.VisitDetectionParametersService; import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.time.Duration; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.List; @@ -21,38 +18,18 @@ public abstract class BaseGoogleTimelineImporter { protected final ObjectMapper objectMapper; protected final ImportBatchProcessor batchProcessor; - private final VisitDetectionParametersService parametersService; public BaseGoogleTimelineImporter(ObjectMapper objectMapper, - ImportBatchProcessor batchProcessor, - VisitDetectionParametersService parametersService) { + ImportBatchProcessor batchProcessor) { this.objectMapper = objectMapper; this.batchProcessor = batchProcessor; - this.parametersService = parametersService; } protected int handleVisit(User user, ZonedDateTime startTime, ZonedDateTime endTime, LatLng latLng, List batch) { - DetectionParameter detectionParameter = parametersService.getCurrentConfiguration(user, startTime.toInstant()); - - logger.info("Found visit at [{}] from start [{}] to end [{}]. Will insert at least [{}] synthetic geo locations.", latLng, startTime, endTime, detectionParameter.getVisitDetection().getMinimumAdjacentPoints()); + logger.info("Found visit at [{}] from start [{}] to end [{}].", latLng, startTime, endTime); createAndScheduleLocationPoint(latLng, startTime, user, batch); - int count = 1; - - long durationBetween = Duration.between(startTime.toInstant(), endTime.toInstant()).toSeconds(); - if (durationBetween > detectionParameter.getVisitDetection().getMinimumStayTimeInSeconds()) { - long increment = 60; - ZonedDateTime currentTime = startTime.plusSeconds(increment); - while (currentTime.isBefore(endTime)) { - createAndScheduleLocationPoint(latLng, currentTime, user, batch); - count+=1; - currentTime = currentTime.plusSeconds(increment); - } - logger.debug("Inserting synthetic points into import to simulate stays at [{}] from [{}] till [{}]", latLng, startTime, endTime); - } else { - logger.info("Skipping creating synthetic points at [{}] since duration was less then [{}] seconds ", latLng, detectionParameter.getVisitDetection().getMinimumStayTimeInSeconds()); - } createAndScheduleLocationPoint(latLng, endTime, user, batch); - return count + 1; + return 2; } protected void createAndScheduleLocationPoint(LatLng latLng, ZonedDateTime timestamp, User user, List batch) { @@ -64,7 +41,7 @@ public abstract class BaseGoogleTimelineImporter { batch.add(point); logger.trace("Created location point at [{}]", point); if (batch.size() >= batchProcessor.getBatchSize()) { - batchProcessor.sendToQueue(user, batch); + batchProcessor.processBatch(user, batch); batch.clear(); } } diff --git a/src/main/java/com/dedicatedcode/reitti/service/importer/GeoJsonImporter.java b/src/main/java/com/dedicatedcode/reitti/service/importer/GeoJsonImporter.java index c5789775..9170f003 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/importer/GeoJsonImporter.java +++ b/src/main/java/com/dedicatedcode/reitti/service/importer/GeoJsonImporter.java @@ -12,7 +12,6 @@ import org.springframework.stereotype.Component; import java.io.IOException; import java.io.InputStream; -import java.sql.Timestamp; import java.time.Instant; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -66,7 +65,7 @@ public class GeoJsonImporter { processedCount.incrementAndGet(); if (batch.size() >= batchProcessor.getBatchSize()) { - batchProcessor.sendToQueue(user, batch); + batchProcessor.processBatch(user, batch); batch.clear(); } } @@ -95,7 +94,7 @@ public class GeoJsonImporter { // Process any remaining locations if (!batch.isEmpty()) { - batchProcessor.sendToQueue(user, batch); + batchProcessor.processBatch(user, batch); } diff --git a/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleAndroidTimelineImporter.java b/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleAndroidTimelineImporter.java index 25d3b2f3..2181b1cd 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleAndroidTimelineImporter.java +++ b/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleAndroidTimelineImporter.java @@ -4,7 +4,6 @@ import com.dedicatedcode.reitti.dto.LocationPoint; import com.dedicatedcode.reitti.model.security.User; import com.dedicatedcode.reitti.service.ImportBatchProcessor; import com.dedicatedcode.reitti.service.ImportStateHolder; -import com.dedicatedcode.reitti.service.VisitDetectionParametersService; import com.dedicatedcode.reitti.service.importer.dto.GoogleTimelineData; import com.dedicatedcode.reitti.service.importer.dto.SemanticSegment; import com.dedicatedcode.reitti.service.importer.dto.TimelinePathPoint; @@ -34,9 +33,8 @@ public class GoogleAndroidTimelineImporter extends BaseGoogleTimelineImporter { public GoogleAndroidTimelineImporter(ObjectMapper objectMapper, ImportStateHolder stateHolder, - ImportBatchProcessor batchProcessor, - VisitDetectionParametersService visitDetectionParametersService) { - super(objectMapper, batchProcessor, visitDetectionParametersService); + ImportBatchProcessor batchProcessor) { + super(objectMapper, batchProcessor); this.stateHolder = stateHolder; } @@ -78,7 +76,7 @@ public class GoogleAndroidTimelineImporter extends BaseGoogleTimelineImporter { // Process any remaining locations if (!batch.isEmpty()) { - batchProcessor.sendToQueue(user, batch); + batchProcessor.processBatch(user, batch); } logger.info("Successfully imported and queued {} location points from Google Timeline for user {}", diff --git a/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleIOSTimelineImporter.java b/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleIOSTimelineImporter.java index 2c86fe47..e96ba410 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleIOSTimelineImporter.java +++ b/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleIOSTimelineImporter.java @@ -4,7 +4,6 @@ import com.dedicatedcode.reitti.dto.LocationPoint; import com.dedicatedcode.reitti.model.security.User; import com.dedicatedcode.reitti.service.ImportBatchProcessor; import com.dedicatedcode.reitti.service.ImportStateHolder; -import com.dedicatedcode.reitti.service.VisitDetectionParametersService; import com.dedicatedcode.reitti.service.importer.dto.ios.IOSSemanticSegment; import com.dedicatedcode.reitti.service.importer.dto.ios.IOSVisit; import com.fasterxml.jackson.core.JsonFactory; @@ -32,9 +31,8 @@ public class GoogleIOSTimelineImporter extends BaseGoogleTimelineImporter { public GoogleIOSTimelineImporter(ObjectMapper objectMapper, ImportStateHolder stateHolder, - ImportBatchProcessor batchProcessor, - VisitDetectionParametersService parametersService) { - super(objectMapper, batchProcessor, parametersService); + ImportBatchProcessor batchProcessor) { + super(objectMapper, batchProcessor); this.stateHolder = stateHolder; } @@ -51,10 +49,6 @@ public class GoogleIOSTimelineImporter extends BaseGoogleTimelineImporter { List semanticSegments = objectMapper.readValue(parser, new TypeReference<>() {}); logger.info("Found {} semantic segments", semanticSegments.size()); for (IOSSemanticSegment semanticSegment : semanticSegments) { - //2024-01-01T00:33:18+01:00 - //2024-01-01T00:33:18+01:00 - - ZonedDateTime start = ZonedDateTime.parse(semanticSegment.getStartTime(), DateTimeFormatter.ISO_OFFSET_DATE_TIME).withNano(0); ZonedDateTime end = ZonedDateTime.parse(semanticSegment.getEndTime(), DateTimeFormatter.ISO_OFFSET_DATE_TIME).withNano(0); if (semanticSegment.getVisit() != null) { @@ -78,7 +72,7 @@ public class GoogleIOSTimelineImporter extends BaseGoogleTimelineImporter { // Process any remaining locations if (!batch.isEmpty()) { - batchProcessor.sendToQueue(user, batch); + batchProcessor.processBatch(user, batch); } logger.info("Successfully imported and queued {} location points from Google Timeline for user {}", diff --git a/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleRecordsImporter.java b/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleRecordsImporter.java index 6138cb2b..87c8dc4d 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleRecordsImporter.java +++ b/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleRecordsImporter.java @@ -66,7 +66,7 @@ public class GoogleRecordsImporter { // Process any remaining locations if (!batch.isEmpty()) { - batchProcessor.sendToQueue(user, batch); + batchProcessor.processBatch(user, batch); } logger.info("Successfully imported and queued {} location points from Google Records for user {}", @@ -112,7 +112,7 @@ public class GoogleRecordsImporter { processedCount++; if (batch.size() >= batchProcessor.getBatchSize()) { - batchProcessor.sendToQueue(user, batch); + batchProcessor.processBatch(user, batch); batch.clear(); } } diff --git a/src/main/java/com/dedicatedcode/reitti/service/importer/GpxImporter.java b/src/main/java/com/dedicatedcode/reitti/service/importer/GpxImporter.java index a76b4cef..1cdf440c 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/importer/GpxImporter.java +++ b/src/main/java/com/dedicatedcode/reitti/service/importer/GpxImporter.java @@ -62,7 +62,7 @@ public class GpxImporter { // Process in batches to avoid memory issues if (batch.size() >= batchProcessor.getBatchSize()) { - batchProcessor.sendToQueue(user, batch); + batchProcessor.processBatch(user, batch); batch.clear(); } } @@ -74,7 +74,7 @@ public class GpxImporter { // Process any remaining locations if (!batch.isEmpty()) { - batchProcessor.sendToQueue(user, batch); + batchProcessor.processBatch(user, batch); } logger.info("Successfully imported and queued {} location points from GPX file for user {}", diff --git a/src/main/java/com/dedicatedcode/reitti/service/integration/OwnTracksRecorderIntegrationService.java b/src/main/java/com/dedicatedcode/reitti/service/integration/OwnTracksRecorderIntegrationService.java index 86c1921e..867c0adc 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/integration/OwnTracksRecorderIntegrationService.java +++ b/src/main/java/com/dedicatedcode/reitti/service/integration/OwnTracksRecorderIntegrationService.java @@ -89,7 +89,7 @@ public class OwnTracksRecorderIntegrationService { } if (!validPoints.isEmpty()) { - importBatchProcessor.sendToQueue(user, validPoints); + importBatchProcessor.processBatch(user, validPoints); totalLocationPoints += validPoints.size(); logger.info("Imported {} location points for user {}", validPoints.size(), user.getUsername()); @@ -236,7 +236,7 @@ public class OwnTracksRecorderIntegrationService { } if (!validPoints.isEmpty()) { - importBatchProcessor.sendToQueue(user, validPoints); + importBatchProcessor.processBatch(user, validPoints); totalLocationPoints += validPoints.size(); logger.debug("Loaded {} location points for user {} from month {}", validPoints.size(), user.getUsername(), month); diff --git a/src/main/java/com/dedicatedcode/reitti/service/processing/LocationDataDensityNormalizer.java b/src/main/java/com/dedicatedcode/reitti/service/processing/LocationDataDensityNormalizer.java new file mode 100644 index 00000000..838e8ef6 --- /dev/null +++ b/src/main/java/com/dedicatedcode/reitti/service/processing/LocationDataDensityNormalizer.java @@ -0,0 +1,330 @@ +package com.dedicatedcode.reitti.service.processing; + +import com.dedicatedcode.reitti.config.LocationDensityConfig; +import com.dedicatedcode.reitti.dto.LocationPoint; +import com.dedicatedcode.reitti.model.geo.RawLocationPoint; +import com.dedicatedcode.reitti.model.processing.DetectionParameter; +import com.dedicatedcode.reitti.model.security.User; +import com.dedicatedcode.reitti.repository.RawLocationPointJdbcService; +import com.dedicatedcode.reitti.service.VisitDetectionParametersService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.time.Instant; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; + +@Service +public class LocationDataDensityNormalizer { + + private static final Logger logger = LoggerFactory.getLogger(LocationDataDensityNormalizer.class); + + private final LocationDensityConfig config; + private final RawLocationPointJdbcService rawLocationPointService; + private final SyntheticLocationPointGenerator syntheticGenerator; + private final VisitDetectionParametersService visitDetectionParametersService; + private final ConcurrentHashMap userLocks = new ConcurrentHashMap<>(); + + @Autowired + public LocationDataDensityNormalizer( + LocationDensityConfig config, + RawLocationPointJdbcService rawLocationPointService, + SyntheticLocationPointGenerator syntheticGenerator, + VisitDetectionParametersService visitDetectionParametersService) { + this.config = config; + this.rawLocationPointService = rawLocationPointService; + this.syntheticGenerator = syntheticGenerator; + this.visitDetectionParametersService = visitDetectionParametersService; + } + + public void normalize(User user, List newPoints) { + if (newPoints == null || newPoints.isEmpty()) { + logger.trace("No points to normalize for user {}", user.getUsername()); + return; + } + + ReentrantLock userLock = userLocks.computeIfAbsent(user.getUsername(), k -> new ReentrantLock()); + + userLock.lock(); + try { + logger.debug("Starting batch density normalization for {} points for user {}", + newPoints.size(), user.getUsername()); + + // Step 1: Compute the time range that encompasses all new points + TimeRange inputRange = computeTimeRange(newPoints); + + // Step 2: Get detection parameters (use the earliest point's time for config lookup) + DetectionParameter detectionParams = visitDetectionParametersService.getCurrentConfiguration(user, inputRange.start); + DetectionParameter.LocationDensity densityConfig = detectionParams.getLocationDensity(); + + // Step 3: Expand the time range by the interpolation window to catch boundary gaps + Duration window = Duration.ofMinutes(densityConfig.getMaxInterpolationGapMinutes()); + TimeRange expandedRange = new TimeRange( + inputRange.start.minus(window), + inputRange.end.plus(window) + ); + + // Step 4: Delete all synthetic points in the expanded range + rawLocationPointService.deleteSyntheticPointsInRange(user, expandedRange.start, expandedRange.end); + + // Step 5: Fetch all existing points in the expanded range (single DB query) + List existingPoints = rawLocationPointService + .findByUserAndTimestampBetweenOrderByTimestampAsc(user, expandedRange.start, expandedRange.end); + + logger.debug("Found {} existing points in expanded range [{} - {}]", + existingPoints.size(), expandedRange.start, expandedRange.end); + + // Step 7: Sort deterministically by timestamp, then by ID (for repeatability) + existingPoints.sort(Comparator + .comparing(RawLocationPoint::getTimestamp) + .thenComparing(p -> p.getGeom().latitude()) + .thenComparing(p -> p.getGeom().longitude()) + .thenComparing(RawLocationPoint::isSynthetic)); + + logger.trace("Processing {} total points after merge", existingPoints.size()); + + // Step 8: Process gaps (generate synthetic points) + processGaps(user, existingPoints, densityConfig); + + // Step 9: Re-fetch and handle excess density + // We need to re-fetch because synthetic points were just inserted + List updatedPoints = rawLocationPointService + .findByUserAndTimestampBetweenOrderByTimestampAsc(user, expandedRange.start, expandedRange.end); + + updatedPoints.sort(Comparator + .comparing(RawLocationPoint::getTimestamp) + .thenComparing(p -> p.getGeom().latitude()) + .thenComparing(p -> p.getGeom().longitude()) + .thenComparing(RawLocationPoint::isSynthetic)); + + handleExcessDensity(user, updatedPoints); + + logger.debug("Completed batch density normalization for user {}", user.getUsername()); + + } catch (Exception e) { + logger.error("Error during batch density normalization for user {}: {}", + user.getUsername(), e.getMessage(), e); + } finally { + userLock.unlock(); + } + } + + /** + * Computes the minimal time range that encompasses all given points. + */ + private TimeRange computeTimeRange(List points) { + Instant minTime = null; + Instant maxTime = null; + + for (LocationPoint point : points) { + Instant timestamp = Instant.parse(point.getTimestamp()); + if (minTime == null || timestamp.isBefore(minTime)) { + minTime = timestamp; + } + if (maxTime == null || timestamp.isAfter(maxTime)) { + maxTime = timestamp; + } + } + + return new TimeRange(minTime, maxTime); + } + + /** + * Processes gaps between points and generates synthetic points where needed. + * Only processes each gap once, regardless of how many input points touch it. + */ + private void processGaps( + User user, + List points, + DetectionParameter.LocationDensity densityConfig) { + + if (points.size() < 2) { + return; + } + + int gapThresholdSeconds = config.getGapThresholdSeconds(); + long maxInterpolationSeconds = densityConfig.getMaxInterpolationGapMinutes() * 60L; + + List allSyntheticPoints = new ArrayList<>(); + Set processedGaps = new HashSet<>(); + + for (int i = 0; i < points.size() - 1; i++) { + RawLocationPoint current = points.get(i); + RawLocationPoint next = points.get(i + 1); + + // Skip if either point is already ignored + if (current.isIgnored() || next.isIgnored()) { + continue; + } + + // Create a deterministic gap key to avoid reprocessing + GapKey gapKey = new GapKey(current.getTimestamp(), next.getTimestamp()); + if (processedGaps.contains(gapKey)) { + continue; + } + processedGaps.add(gapKey); + + long gapSeconds = Duration.between(current.getTimestamp(), next.getTimestamp()).getSeconds(); + + if (gapSeconds > gapThresholdSeconds && gapSeconds <= maxInterpolationSeconds) { + logger.trace("Found gap of {} seconds between {} and {}", + gapSeconds, current.getTimestamp(), next.getTimestamp()); + + List syntheticPoints = syntheticGenerator.generateSyntheticPoints( + current, + next, + config.getTargetPointsPerMinute(), + densityConfig.getMaxInterpolationDistanceMeters() + ); + + allSyntheticPoints.addAll(syntheticPoints); + } + } + + if (!allSyntheticPoints.isEmpty()) { + // Sort synthetic points by timestamp for deterministic insertion order + allSyntheticPoints.sort(Comparator.comparing(LocationPoint::getTimestamp)); + + int inserted = rawLocationPointService.bulkInsertSynthetic(user, allSyntheticPoints); + logger.debug("Inserted {} synthetic points for user {}", inserted, user.getUsername()); + } + } + + /** + * Handles excess density by marking redundant points as ignored. + * Uses deterministic rules for selecting which point to ignore. + */ + private void handleExcessDensity(User user, List points) { + if (points.size() < 2) { + return; + } + + int toleranceSeconds = config.getToleranceSeconds(); + Set pointsToIgnore = new LinkedHashSet<>(); // Preserve order for debugging + Set alreadyConsidered = new HashSet<>(); + + for (int i = 0; i < points.size() - 1; i++) { + RawLocationPoint current = points.get(i); + RawLocationPoint next = points.get(i + 1); + + // Skip points without IDs (not persisted) or already ignored + if (current.getId() == null || next.getId() == null) { + continue; + } + if (current.isIgnored() || next.isIgnored()) { + continue; + } + if (alreadyConsidered.contains(current.getId()) || alreadyConsidered.contains(next.getId())) { + continue; + } + + long timeDiff = Duration.between(current.getTimestamp(), next.getTimestamp()).getSeconds(); + + if (timeDiff < toleranceSeconds) { + RawLocationPoint toIgnore = selectPointToIgnore(current, next); + + if (toIgnore != null && toIgnore.getId() != null) { + pointsToIgnore.add(toIgnore.getId()); + alreadyConsidered.add(toIgnore.getId()); + logger.trace("Marking point {} as ignored due to excess density", toIgnore.getId()); + } + } + } + + if (!pointsToIgnore.isEmpty()) { + rawLocationPointService.bulkUpdateIgnoredStatus(new ArrayList<>(pointsToIgnore), true); + logger.debug("Marked {} points as ignored for user {}", pointsToIgnore.size(), user.getUsername()); + } + } + + /** + * Selects which point to ignore when two points are too close together. + * Rules (in priority order): + * 1. Prefer real points over synthetic points + * 2. Prefer points with better accuracy (lower accuracy value) + * 3. Prefer points with accuracy info over those without + * 4. Prefer points with lower ID (earlier insertion = more authoritative) + */ + private RawLocationPoint selectPointToIgnore(RawLocationPoint point1, RawLocationPoint point2) { + // Rule 1: Never ignore real points if the other is synthetic + if (!point1.isSynthetic() && point2.isSynthetic()) { + return point2; + } + if (point1.isSynthetic() && !point2.isSynthetic()) { + return point1; + } + + // Rule 2 & 3: Prefer points with better accuracy + Double acc1 = point1.getAccuracyMeters(); + Double acc2 = point2.getAccuracyMeters(); + + if (acc1 != null && acc2 != null) { + if (!acc1.equals(acc2)) { + return acc1 < acc2 ? point2 : point1; + } + } else if (acc1 != null) { + return point2; + } else if (acc2 != null) { + return point1; + } + + int timestampCompare = point1.getTimestamp().compareTo(point2.getTimestamp()); + if (timestampCompare != 0) { + return timestampCompare < 0 ? point2 : point1; + } + + // Tiebreaker: use coordinates (immutable, deterministic) + int latCompare = Double.compare(point1.getGeom().latitude(), point2.getGeom().latitude()); + if (latCompare != 0) { + return latCompare < 0 ? point2 : point1; + } + + int lonCompare = Double.compare(point1.getGeom().longitude(), point2.getGeom().longitude()); + return lonCompare < 0 ? point2 : point1; + + } + + /** + * Represents a time range with start and end instants. + */ + private static class TimeRange { + final Instant start; + final Instant end; + + TimeRange(Instant start, Instant end) { + this.start = start; + this.end = end; + } + } + + /** + * Represents a unique gap between two timestamps. + * Used to avoid processing the same gap multiple times. + */ + private static class GapKey { + final Instant start; + final Instant end; + + GapKey(Instant start, Instant end) { + this.start = start; + this.end = end; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GapKey gapKey = (GapKey) o; + return Objects.equals(start, gapKey.start) && Objects.equals(end, gapKey.end); + } + + @Override + public int hashCode() { + return Objects.hash(start, end); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/dedicatedcode/reitti/service/processing/LocationDataIngestPipeline.java b/src/main/java/com/dedicatedcode/reitti/service/processing/LocationDataIngestPipeline.java index 9118012b..e7f02070 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/processing/LocationDataIngestPipeline.java +++ b/src/main/java/com/dedicatedcode/reitti/service/processing/LocationDataIngestPipeline.java @@ -7,6 +7,7 @@ import com.dedicatedcode.reitti.repository.RawLocationPointJdbcService; import com.dedicatedcode.reitti.repository.UserJdbcService; import com.dedicatedcode.reitti.repository.UserSettingsJdbcService; import com.dedicatedcode.reitti.service.UserNotificationService; +import jakarta.annotation.PreDestroy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -24,36 +25,54 @@ public class LocationDataIngestPipeline { private final RawLocationPointJdbcService rawLocationPointJdbcService; private final UserSettingsJdbcService userSettingsJdbcService; private final UserNotificationService userNotificationService; + private final LocationDataDensityNormalizer densityNormalizer; @Autowired public LocationDataIngestPipeline(GeoPointAnomalyFilter geoPointAnomalyFilter, UserJdbcService userJdbcService, RawLocationPointJdbcService rawLocationPointJdbcService, UserSettingsJdbcService userSettingsJdbcService, - UserNotificationService userNotificationService) { + UserNotificationService userNotificationService, + LocationDataDensityNormalizer densityNormalizer) { this.geoPointAnomalyFilter = geoPointAnomalyFilter; this.userJdbcService = userJdbcService; this.rawLocationPointJdbcService = rawLocationPointJdbcService; this.userSettingsJdbcService = userSettingsJdbcService; this.userNotificationService = userNotificationService; + this.densityNormalizer = densityNormalizer; + } + + @PreDestroy + public void shutdown() { } public void processLocationData(LocationDataEvent event) { - long start = System.currentTimeMillis(); + try { + long start = System.currentTimeMillis(); + logger.debug("starting processing of event: {}", event); - Optional userOpt = userJdbcService.findByUsername(event.getUsername()); + Optional userOpt = userJdbcService.findByUsername(event.getUsername()); - if (userOpt.isEmpty()) { - logger.warn("User not found for name: [{}]", event.getUsername()); - return; + if (userOpt.isEmpty()) { + logger.warn("User not found for name: [{}]", event.getUsername()); + return; + } + + User user = userOpt.get(); + List points = event.getPoints(); + List filtered = this.geoPointAnomalyFilter.filterAnomalies(points); + + // Store real points first + int updatedRows = rawLocationPointJdbcService.bulkInsert(user, filtered); + + // Normalize density around each new point + densityNormalizer.normalize(user, filtered); + + userSettingsJdbcService.updateNewestData(user, filtered); + userNotificationService.newRawLocationData(user, filtered); + logger.info("Finished storing and normalizing points [{}] for user [{}] in [{}]ms. Filtered out [{}] points before database and [{}] after database.", filtered.size(), event.getUsername(), System.currentTimeMillis() - start, points.size() - filtered.size(), filtered.size() - updatedRows); + } catch (Exception e) { + logger.error("Error during processing of event: {}", event, e); } - - User user = userOpt.get(); - List points = event.getPoints(); - List filtered = this.geoPointAnomalyFilter.filterAnomalies(points); - int updatedRows = rawLocationPointJdbcService.bulkInsert(user, filtered); - userSettingsJdbcService.updateNewestData(user, filtered); - userNotificationService.newRawLocationData(user, filtered); - logger.info("Finished storing points [{}] for user [{}] in [{}]ms. Filtered out [{}] points before database and [{}] after database.", filtered.size(), event.getUsername(), System.currentTimeMillis() - start, points.size() - filtered.size(), filtered.size() - updatedRows); } } diff --git a/src/main/java/com/dedicatedcode/reitti/service/processing/ProcessingPipelineTrigger.java b/src/main/java/com/dedicatedcode/reitti/service/processing/ProcessingPipelineTrigger.java index a19e11b1..655364fa 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/processing/ProcessingPipelineTrigger.java +++ b/src/main/java/com/dedicatedcode/reitti/service/processing/ProcessingPipelineTrigger.java @@ -1,6 +1,5 @@ package com.dedicatedcode.reitti.service.processing; -import com.dedicatedcode.reitti.config.RabbitMQConfig; import com.dedicatedcode.reitti.event.LocationProcessEvent; import com.dedicatedcode.reitti.event.TriggerProcessingEvent; import com.dedicatedcode.reitti.model.geo.RawLocationPoint; @@ -12,24 +11,30 @@ import com.dedicatedcode.reitti.service.ImportStateHolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import java.time.Instant; import java.util.List; import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicBoolean; @Service public class ProcessingPipelineTrigger { private static final Logger log = LoggerFactory.getLogger(ProcessingPipelineTrigger.class); - private static final int BATCH_SIZE = 100; private final ImportStateHolder stateHolder; private final RawLocationPointJdbcService rawLocationPointJdbcService; private final PreviewRawLocationPointJdbcService previewRawLocationPointJdbcService; private final UserJdbcService userJdbcService; - private final RabbitTemplate rabbitTemplate; + private final UnifiedLocationProcessingService unifiedLocationProcessingService; + private final int batchSize; + private final ThreadPoolExecutor executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); private final AtomicBoolean isRunning = new AtomicBoolean(false); @@ -37,53 +42,49 @@ public class ProcessingPipelineTrigger { RawLocationPointJdbcService rawLocationPointJdbcService, PreviewRawLocationPointJdbcService previewRawLocationPointJdbcService, UserJdbcService userJdbcService, - RabbitTemplate rabbitTemplate) { + RabbitTemplate rabbitTemplate, UnifiedLocationProcessingService unifiedLocationProcessingService, + @Value("${reitti.import.batch-size:100}") int batchSize) { this.stateHolder = stateHolder; this.rawLocationPointJdbcService = rawLocationPointJdbcService; this.previewRawLocationPointJdbcService = previewRawLocationPointJdbcService; this.userJdbcService = userJdbcService; - this.rabbitTemplate = rabbitTemplate; + this.unifiedLocationProcessingService = unifiedLocationProcessingService; + this.batchSize = batchSize; } @Scheduled(cron = "${reitti.process-data.schedule}") public void start() { - if (isBusy()) return; - - isRunning.set(true); - try { - for (User user : userJdbcService.findAll()) { - handleDataForUser(user, null); - } - } finally { - isRunning.set(false); + if (stateHolder.isImportRunning()) { + log.warn("Data Import is currently running, wil skip this run"); + return; } + for (User user : userJdbcService.findAll()) { + handleDataForUser(user, null, UUID.randomUUID().toString()); + } + } + + public void start(User user) { + handleDataForUser(user, null, UUID.randomUUID().toString()); } public void handle(TriggerProcessingEvent event) { - if (isBusy()) return; - - isRunning.set(true); - try { - Optional byUsername = this.userJdbcService.findByUsername(event.getUsername()); - if (byUsername.isPresent()) { - handleDataForUser(byUsername.get(), event.getPreviewId()); - } else { - log.warn("No user found for username: {}", event.getUsername()); - } - } finally { - isRunning.set(false); + Optional byUsername = this.userJdbcService.findByUsername(event.getUsername()); + if (byUsername.isPresent()) { + handleDataForUser(byUsername.get(), event.getPreviewId(), event.getTraceId()); + } else { + log.warn("No user found for username: {}", event.getUsername()); } } - private void handleDataForUser(User user, String previewId) { + private void handleDataForUser(User user, String previewId, String traceId) { int totalProcessed = 0; - + while (true) { List currentBatch; if (previewId == null) { - currentBatch = rawLocationPointJdbcService.findByUserAndProcessedIsFalseOrderByTimestampWithLimit(user, BATCH_SIZE, 0); + currentBatch = rawLocationPointJdbcService.findByUserAndProcessedIsFalseOrderByTimestampWithLimit(user, batchSize, 0); } else { - currentBatch = previewRawLocationPointJdbcService.findByUserAndProcessedIsFalseOrderByTimestampWithLimit(user, previewId, BATCH_SIZE, 0); + currentBatch = previewRawLocationPointJdbcService.findByUserAndProcessedIsFalseOrderByTimestampWithLimit(user, previewId, batchSize, 0); } if (currentBatch.isEmpty()) { @@ -100,29 +101,16 @@ public class ProcessingPipelineTrigger { } else { rawLocationPointJdbcService.bulkUpdateProcessedStatus(currentBatch); } - - this.rabbitTemplate - .convertAndSend(RabbitMQConfig.EXCHANGE_NAME, - RabbitMQConfig.STAY_DETECTION_ROUTING_KEY, - new LocationProcessEvent(user.getUsername(), earliest, latest, previewId)); - + + executorService.submit(() -> unifiedLocationProcessingService.processLocationEvent(new LocationProcessEvent(user.getUsername(), earliest, latest, previewId, traceId))); totalProcessed += currentBatch.size(); } - + log.debug("Processed [{}] unprocessed points for user [{}]", totalProcessed, user.getId()); } - private boolean isBusy() { - if (isRunning.get()) { - log.warn("Processing is already running, wil skip this run"); - return true; - } - - if (stateHolder.isImportRunning()) { - log.warn("Data Import is currently running, wil skip this run"); - return true; - } - return false; + public boolean isIdle() { + return executorService.getQueue().isEmpty() && + executorService.getActiveCount() == 0; } - } diff --git a/src/main/java/com/dedicatedcode/reitti/service/processing/SyntheticLocationPointGenerator.java b/src/main/java/com/dedicatedcode/reitti/service/processing/SyntheticLocationPointGenerator.java new file mode 100644 index 00000000..1b615c47 --- /dev/null +++ b/src/main/java/com/dedicatedcode/reitti/service/processing/SyntheticLocationPointGenerator.java @@ -0,0 +1,126 @@ +package com.dedicatedcode.reitti.service.processing; + +import com.dedicatedcode.reitti.dto.LocationPoint; +import com.dedicatedcode.reitti.model.geo.GeoPoint; +import com.dedicatedcode.reitti.model.geo.GeoUtils; +import com.dedicatedcode.reitti.model.geo.RawLocationPoint; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; + +import static java.time.temporal.ChronoUnit.SECONDS; + +@Service +public class SyntheticLocationPointGenerator { + + private static final Logger logger = LoggerFactory.getLogger(SyntheticLocationPointGenerator.class); + + public List generateSyntheticPoints( + RawLocationPoint startPoint, + RawLocationPoint endPoint, + int targetPointsPerMinute, + double maxDistanceMeters) { + + if (!shouldInterpolate(startPoint, endPoint, maxDistanceMeters)) { + logger.trace("Skipping interpolation between points: distance too large or other constraints not met"); + return List.of(); + } + + List syntheticPoints = new ArrayList<>(); + + // Calculate target interval in seconds + int intervalSeconds = 60 / targetPointsPerMinute; + + Instant startTime = startPoint.getTimestamp(); + Instant endTime = endPoint.getTimestamp(); + + // Generate points at regular intervals, excluding the endpoints + Instant currentTime = startTime.plusSeconds(intervalSeconds).truncatedTo(SECONDS); + + while (currentTime.isBefore(endTime)) { + // Calculate interpolation ratio (0.0 to 1.0) + long totalDuration = endTime.getEpochSecond() - startTime.getEpochSecond(); + long currentDuration = currentTime.getEpochSecond() - startTime.getEpochSecond(); + double ratio = (double) currentDuration / totalDuration; + + // Interpolate coordinates + GeoPoint interpolatedCoords = interpolateCoordinates( + startPoint.getGeom(), + endPoint.getGeom(), + ratio + ); + + // Interpolate accuracy and elevation + Double interpolatedAccuracy = interpolateValue( + startPoint.getAccuracyMeters(), + endPoint.getAccuracyMeters(), + ratio + ); + + Double interpolatedElevation = interpolateValue( + startPoint.getElevationMeters(), + endPoint.getElevationMeters(), + ratio + ); + + // Create synthetic LocationPoint + LocationPoint syntheticPoint = new LocationPoint(); + syntheticPoint.setLatitude(interpolatedCoords.latitude()); + syntheticPoint.setLongitude(interpolatedCoords.longitude()); + syntheticPoint.setTimestamp(currentTime.atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + syntheticPoint.setAccuracyMeters(interpolatedAccuracy); + syntheticPoint.setElevationMeters(interpolatedElevation); + + syntheticPoints.add(syntheticPoint); + + currentTime = currentTime.plusSeconds(intervalSeconds); + } + + logger.trace("Generated {} synthetic points between {} and {}", + syntheticPoints.size(), startTime, endTime); + + return syntheticPoints; + } + + private boolean shouldInterpolate(RawLocationPoint start, RawLocationPoint end, double maxDistance) { + // Check if distance between points is within acceptable range + double distance = GeoUtils.distanceInMeters(start, end); + + if (distance > maxDistance) { + logger.trace("Distance {} meters exceeds maximum interpolation distance {} meters", + distance, maxDistance); + return false; + } + + return true; + } + + private GeoPoint interpolateCoordinates(GeoPoint start, GeoPoint end, double ratio) { + // Use linear interpolation for coordinates + // For more accuracy over long distances, could use great circle interpolation + double lat = start.latitude() + (end.latitude() - start.latitude()) * ratio; + double lon = start.longitude() + (end.longitude() - start.longitude()) * ratio; + + return new GeoPoint(lat, lon); + } + + private Double interpolateValue(Double start, Double end, double ratio) { + if (start == null && end == null) { + return null; + } + if (start == null) { + return end; + } + if (end == null) { + return start; + } + + return start + (end - start) * ratio; + } +} diff --git a/src/main/java/com/dedicatedcode/reitti/service/processing/TripDetectionService.java b/src/main/java/com/dedicatedcode/reitti/service/processing/TripDetectionService.java deleted file mode 100644 index a900d953..00000000 --- a/src/main/java/com/dedicatedcode/reitti/service/processing/TripDetectionService.java +++ /dev/null @@ -1,201 +0,0 @@ -package com.dedicatedcode.reitti.service.processing; - -import com.dedicatedcode.reitti.event.ProcessedVisitCreatedEvent; -import com.dedicatedcode.reitti.event.RecalculateTripEvent; -import com.dedicatedcode.reitti.model.geo.*; -import com.dedicatedcode.reitti.model.security.User; -import com.dedicatedcode.reitti.repository.*; -import com.dedicatedcode.reitti.service.UserNotificationService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantLock; - -@Service -public class TripDetectionService { - - private static final Logger logger = LoggerFactory.getLogger(TripDetectionService.class); - - private final ProcessedVisitJdbcService processedVisitJdbcService; - private final PreviewProcessedVisitJdbcService previewProcessedVisitJdbcService; - private final RawLocationPointJdbcService rawLocationPointJdbcService; - private final PreviewRawLocationPointJdbcService previewRawLocationPointJdbcService; - private final TripJdbcService tripJdbcService; - private final PreviewTripJdbcService previewTripJdbcService; - private final TransportModeService transportModeService; - private final UserJdbcService userJdbcService; - private final UserNotificationService userNotificationService; - private final ConcurrentHashMap userLocks = new ConcurrentHashMap<>(); - - public TripDetectionService(ProcessedVisitJdbcService processedVisitJdbcService, - PreviewProcessedVisitJdbcService previewProcessedVisitJdbcService, - RawLocationPointJdbcService rawLocationPointJdbcService, - PreviewRawLocationPointJdbcService previewRawLocationPointJdbcService, - TripJdbcService tripJdbcService, - PreviewTripJdbcService previewTripJdbcService, TransportModeService transportModeService, - UserJdbcService userJdbcService, - UserNotificationService userNotificationService) { - this.processedVisitJdbcService = processedVisitJdbcService; - this.previewProcessedVisitJdbcService = previewProcessedVisitJdbcService; - this.rawLocationPointJdbcService = rawLocationPointJdbcService; - this.previewRawLocationPointJdbcService = previewRawLocationPointJdbcService; - this.tripJdbcService = tripJdbcService; - this.previewTripJdbcService = previewTripJdbcService; - this.transportModeService = transportModeService; - this.userJdbcService = userJdbcService; - this.userNotificationService = userNotificationService; - } - - public void visitCreated(ProcessedVisitCreatedEvent event) { - String username = event.getUsername(); - ReentrantLock userLock = userLocks.computeIfAbsent(username, k -> new ReentrantLock()); - - userLock.lock(); - try { - User user = this.userJdbcService.findByUsername(username).orElseThrow(); - - Optional createdVisit; - if (event.getPreviewId() == null) { - createdVisit = this.processedVisitJdbcService.findByUserAndId(user, event.getVisitId()); - } else { - createdVisit = this.previewProcessedVisitJdbcService.findByUserAndId(user, event.getVisitId()); - } - createdVisit.ifPresent(visit -> { - //find visits in timerange - Instant searchStart = visit.getStartTime().minus(1, ChronoUnit.DAYS); - Instant searchEnd = visit.getEndTime().plus(1, ChronoUnit.DAYS); - - List visits; - if (event.getPreviewId() == null) { - visits = this.processedVisitJdbcService.findByUserAndTimeOverlap(user, searchStart, searchEnd); - } else { - visits = this.previewProcessedVisitJdbcService.findByUserAndTimeOverlap(user, event.getPreviewId(), searchStart, searchEnd); - } - - if (visits.size() < 2) { - logger.info("Not enough visits to detect trips for user: {}", user.getUsername()); - return; - } - - List trips = new ArrayList<>(); - // Iterate through consecutive visits to detect trips - for (int i = 0; i < visits.size() - 1; i++) { - ProcessedVisit startVisit = visits.get(i); - ProcessedVisit endVisit = visits.get(i + 1); - - // Create a trip between these two visits - Trip trip = createTripBetweenVisits(user, event.getPreviewId(), startVisit, endVisit); - if (trip != null) { - trips.add(trip); - } - } - - if (event.getPreviewId() == null) { - tripJdbcService.bulkInsert(user, trips); - } else { - previewTripJdbcService.bulkInsert(user, event.getPreviewId(), trips); - - } - if (event.getPreviewId() == null) { - userNotificationService.newTrips(user, trips); - } else { - userNotificationService.newTrips(user, trips, event.getPreviewId()); - } - }); - } finally { - userLock.unlock(); - } - } - - private Trip createTripBetweenVisits(User user, String previewId, ProcessedVisit startVisit, ProcessedVisit endVisit) { - // Trip starts when the first visit ends - Instant tripStartTime = startVisit.getEndTime(); - - // Trip ends when the second visit starts - Instant tripEndTime = endVisit.getStartTime(); - - if (previewId != null) { - if (this.previewProcessedVisitJdbcService.findById(startVisit.getId()).isEmpty() || this.previewProcessedVisitJdbcService.findById(endVisit.getId()).isEmpty()) { - logger.debug("One of the following preview visits [{},{}] where already deleted. Will skip trip creation.", startVisit.getId(), endVisit.getId()); - return null; - } - } else { - if (this.processedVisitJdbcService.findById(startVisit.getId()).isEmpty() || this.processedVisitJdbcService.findById(endVisit.getId()).isEmpty()) { - logger.debug("One of the following visits [{},{}] where already deleted. Will skip trip creation.", startVisit.getId(), endVisit.getId()); - return null; - } - } - // If end time is before or equal to start time, this is not a valid trip - if (tripEndTime.isBefore(tripStartTime) || tripEndTime.equals(tripStartTime)) { - logger.warn("Invalid trip time range detected for user {}: {} to {}", - user.getUsername(), tripStartTime, tripEndTime); - return null; - } - - - if (previewId == null) { - // Check if a trip already exists with the same start and end times - if (tripJdbcService.existsByUserAndStartTimeAndEndTime(user, tripStartTime, tripEndTime)) { - logger.debug("Trip already exists for user {} from {} to {}", - user.getUsername(), tripStartTime, tripEndTime); - return null; - } - } - - // Get location points between the two visits - List tripPoints; - if (previewId == null) { - tripPoints = rawLocationPointJdbcService.findByUserAndTimestampBetweenOrderByTimestampAsc(user, tripStartTime, tripEndTime); - } else { - tripPoints = previewRawLocationPointJdbcService.findByUserAndTimestampBetweenOrderByTimestampAsc(user, previewId, tripStartTime, tripEndTime); - } - double estimatedDistanceInMeters = calculateDistanceBetweenPlaces(startVisit.getPlace(), endVisit.getPlace()); - double travelledDistanceMeters = GeoUtils.calculateTripDistance(tripPoints); - // Create a new trip - TransportMode transportMode = this.transportModeService.inferTransportMode(user, tripPoints, tripStartTime, tripEndTime); - Trip trip = new Trip( - tripStartTime, - tripEndTime, - tripEndTime.getEpochSecond() - tripStartTime.getEpochSecond(), - estimatedDistanceInMeters, - travelledDistanceMeters, - transportMode, - startVisit, - endVisit - ); - logger.debug("Created trip from {} to {}: travelled distance={}m, mode={}", - startVisit.getPlace().getName(), endVisit.getPlace().getName(), Math.round(travelledDistanceMeters), transportMode); - - // Save and return the trip - return trip; - } - - private double calculateDistanceBetweenPlaces(SignificantPlace place1, SignificantPlace place2) { - return GeoUtils.distanceInMeters( - place1.getLatitudeCentroid(), place1.getLongitudeCentroid(), - place2.getLatitudeCentroid(), place2.getLongitudeCentroid()); - } - - public void recalculateTrip(RecalculateTripEvent event) { - User user = this.userJdbcService.findByUsername(event.getUsername()).orElseThrow(); - - this.tripJdbcService.findByUserAndId(user, event.getTripId()).ifPresent(trip -> { - logger.info("Recalculating trip [{}] for user [{}]", trip.getId(), user.getUsername()); - Instant startTime = trip.getStartTime(); - Instant endTime = trip.getEndTime(); - List tripPoints = rawLocationPointJdbcService.findByUserAndTimestampBetweenOrderByTimestampAsc(user, startTime, endTime); - TransportMode transportMode = this.transportModeService.inferTransportMode(user, tripPoints, startTime, endTime); - if (trip.getTransportModeInferred() != transportMode) { - logger.info("TransportMode changed from [{}] to [{}] for trip [{}]", trip.getTransportModeInferred(), transportMode, trip.getId()); - this.tripJdbcService.update(trip.withTransportMode(transportMode)); - } - }); - } -} diff --git a/src/main/java/com/dedicatedcode/reitti/service/processing/UnifiedLocationProcessingService.java b/src/main/java/com/dedicatedcode/reitti/service/processing/UnifiedLocationProcessingService.java new file mode 100644 index 00000000..66159b38 --- /dev/null +++ b/src/main/java/com/dedicatedcode/reitti/service/processing/UnifiedLocationProcessingService.java @@ -0,0 +1,824 @@ +package com.dedicatedcode.reitti.service.processing; + +import com.dedicatedcode.reitti.config.RabbitMQConfig; +import com.dedicatedcode.reitti.event.LocationProcessEvent; +import com.dedicatedcode.reitti.event.SignificantPlaceCreatedEvent; +import com.dedicatedcode.reitti.model.ClusteredPoint; +import com.dedicatedcode.reitti.model.PlaceInformationOverride; +import com.dedicatedcode.reitti.model.geo.*; +import com.dedicatedcode.reitti.model.processing.DetectionParameter; +import com.dedicatedcode.reitti.model.security.User; +import com.dedicatedcode.reitti.repository.*; +import com.dedicatedcode.reitti.service.GeoLocationTimezoneService; +import com.dedicatedcode.reitti.service.UserNotificationService; +import com.dedicatedcode.reitti.service.VisitDetectionParametersService; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Point; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Collectors; + +/** + * Unified service that processes the entire GPS pipeline atomically per user. + * Ensures deterministic, repeatable results by processing events sequentially + * per user while maintaining parallelism across different users. + */ +@Service +public class UnifiedLocationProcessingService { + + private static final Logger logger = LoggerFactory.getLogger(UnifiedLocationProcessingService.class); + + private final UserJdbcService userJdbcService; + private final RawLocationPointJdbcService rawLocationPointJdbcService; + private final PreviewRawLocationPointJdbcService previewRawLocationPointJdbcService; + private final VisitJdbcService visitJdbcService; + private final PreviewVisitJdbcService previewVisitJdbcService; + private final ProcessedVisitJdbcService processedVisitJdbcService; + private final PreviewProcessedVisitJdbcService previewProcessedVisitJdbcService; + private final TripJdbcService tripJdbcService; + private final PreviewTripJdbcService previewTripJdbcService; + private final SignificantPlaceJdbcService significantPlaceJdbcService; + private final PreviewSignificantPlaceJdbcService previewSignificantPlaceJdbcService; + private final SignificantPlaceOverrideJdbcService significantPlaceOverrideJdbcService; + private final VisitDetectionParametersService visitDetectionParametersService; + private final PreviewVisitDetectionParametersJdbcService previewVisitDetectionParametersJdbcService; + private final TransportModeService transportModeService; + private final UserNotificationService userNotificationService; + private final GeoLocationTimezoneService timezoneService; + private final GeometryFactory geometryFactory; + private final RabbitTemplate rabbitTemplate; + + public UnifiedLocationProcessingService( + UserJdbcService userJdbcService, + RawLocationPointJdbcService rawLocationPointJdbcService, + PreviewRawLocationPointJdbcService previewRawLocationPointJdbcService, + VisitJdbcService visitJdbcService, + PreviewVisitJdbcService previewVisitJdbcService, + ProcessedVisitJdbcService processedVisitJdbcService, + PreviewProcessedVisitJdbcService previewProcessedVisitJdbcService, + TripJdbcService tripJdbcService, + PreviewTripJdbcService previewTripJdbcService, + SignificantPlaceJdbcService significantPlaceJdbcService, + PreviewSignificantPlaceJdbcService previewSignificantPlaceJdbcService, + SignificantPlaceOverrideJdbcService significantPlaceOverrideJdbcService, + VisitDetectionParametersService visitDetectionParametersService, PreviewVisitDetectionParametersJdbcService previewVisitDetectionParametersJdbcService, + TransportModeService transportModeService, + UserNotificationService userNotificationService, + GeoLocationTimezoneService timezoneService, + GeometryFactory geometryFactory, RabbitTemplate rabbitTemplate) { + + this.userJdbcService = userJdbcService; + this.rawLocationPointJdbcService = rawLocationPointJdbcService; + this.previewRawLocationPointJdbcService = previewRawLocationPointJdbcService; + this.visitJdbcService = visitJdbcService; + this.previewVisitJdbcService = previewVisitJdbcService; + this.processedVisitJdbcService = processedVisitJdbcService; + this.previewProcessedVisitJdbcService = previewProcessedVisitJdbcService; + this.tripJdbcService = tripJdbcService; + this.previewTripJdbcService = previewTripJdbcService; + this.significantPlaceJdbcService = significantPlaceJdbcService; + this.previewSignificantPlaceJdbcService = previewSignificantPlaceJdbcService; + this.significantPlaceOverrideJdbcService = significantPlaceOverrideJdbcService; + this.visitDetectionParametersService = visitDetectionParametersService; + this.previewVisitDetectionParametersJdbcService = previewVisitDetectionParametersJdbcService; + this.transportModeService = transportModeService; + this.userNotificationService = userNotificationService; + this.timezoneService = timezoneService; + this.geometryFactory = geometryFactory; + this.rabbitTemplate = rabbitTemplate; + } + + /** + * Entry point for location processing events. + * Enqueues the event for the user and ensures processing starts. + */ + public void processLocationEvent(LocationProcessEvent event) { + long startTime = System.currentTimeMillis(); + String username = event.getUsername(); + String previewId = event.getPreviewId(); + + logger.info("Processing location data for user [{}], mode: {}", + username, previewId == null ? "LIVE" : "PREVIEW"); + + User user = userJdbcService.findByUsername(username) + .orElseThrow(() -> new IllegalStateException("User not found: " + username)); + + // STEP 1: Visit Detection + // ---------------------- + VisitDetectionResult detectionResult = detectVisits(user, event); + logger.debug("Detection: {} visits created", detectionResult.visits.size()); + + // STEP 2: Visit Merging + // --------------------- + VisitMergingResult mergingResult = mergeVisits( + user, + previewId, + event.getTraceId(), + detectionResult.searchStart, + detectionResult.searchEnd, + detectionResult.visits); + logger.debug("Merging: {} visits merged into {} processed visits", + mergingResult.inputVisits.size(), + mergingResult.processedVisits.size()); + + // STEP 3: Trip Detection + // ---------------------- + TripDetectionResult tripResult = detectTrips( + user, + previewId, + mergingResult.searchStart, + mergingResult.searchEnd + ); + logger.debug("Trip detection: {} trips created", tripResult.trips.size()); + + // STEP 4: Notifications + // --------------------- + if (previewId == null) { + userNotificationService.newVisits(user, mergingResult.processedVisits); + userNotificationService.newTrips(user, tripResult.trips); + } else { + userNotificationService.newTrips(user, tripResult.trips, previewId); + } + + long duration = System.currentTimeMillis() - startTime; + logger.info("Completed processing for user [{}] in {}ms: {} visits → {} processed visits → {} trips", + username, duration, detectionResult.visits.size(), + mergingResult.processedVisits.size(), tripResult.trips.size()); + } + + /** + * STEP 1: Visit Detection + * Detects stay points from raw location data and creates Visit entities. + */ + private VisitDetectionResult detectVisits(User user, LocationProcessEvent event) { + String previewId = event.getPreviewId(); + Instant windowStart = event.getEarliest().minus(1, ChronoUnit.DAYS); + Instant windowEnd = event.getLatest().plus(1, ChronoUnit.DAYS); + + // Get detection parameters + DetectionParameter currentConfiguration; + DetectionParameter.VisitDetection detectionParams; + if (previewId == null) { + currentConfiguration = visitDetectionParametersService.getCurrentConfiguration(user, windowStart); + } else { + currentConfiguration = previewVisitDetectionParametersJdbcService.findCurrent(user, previewId); + } + detectionParams = currentConfiguration.getVisitDetection(); + + // Find and delete affected visits + List affectedVisits; + if (previewId == null) { + affectedVisits = visitJdbcService.findByUserAndTimeAfterAndStartTimeBefore( + user, windowStart, windowEnd); + visitJdbcService.delete(affectedVisits); + } else { + affectedVisits = previewVisitJdbcService.findByUserAndTimeAfterAndStartTimeBefore( + user, previewId, windowStart, windowEnd); + previewVisitJdbcService.delete(affectedVisits); + } + + // Expand window based on deleted visits + if (!affectedVisits.isEmpty()) { + if (affectedVisits.getFirst().getStartTime().isBefore(windowStart)) { + windowStart = affectedVisits.getFirst().getStartTime(); + } + if (affectedVisits.getLast().getEndTime().isAfter(windowEnd)) { + windowEnd = affectedVisits.getLast().getEndTime(); + } + } + + // Get clustered points + double baseLatitude = affectedVisits.isEmpty() ? 50 : affectedVisits.getFirst().getLatitude(); + double metersAsDegrees = GeoUtils.metersToDegreesAtPosition((double) currentConfiguration.getVisitMerging().getMinDistanceBetweenVisits() / 2, baseLatitude); + + List clusteredPoints; + int minimumAdjacentPoints = Math.max(4, Math.toIntExact(detectionParams.getMinimumStayTimeInSeconds() / 20)); + if (previewId == null) { + clusteredPoints = rawLocationPointJdbcService.findClusteredPointsInTimeRangeForUser( + user, windowStart, windowEnd, minimumAdjacentPoints, metersAsDegrees); + } else { + clusteredPoints = previewRawLocationPointJdbcService.findClusteredPointsInTimeRangeForUser( + user, previewId, windowStart, windowEnd, minimumAdjacentPoints, metersAsDegrees); + } + logger.debug("Searching for clustered points in range [{}, {}], minimum adjacent points: {} ", windowStart, windowEnd, minimumAdjacentPoints); + + // Cluster by location and time + Map> clusteredByLocation = new TreeMap<>(); + for (ClusteredPoint cp : clusteredPoints.stream().filter(clusteredPoint -> !clusteredPoint.getPoint().isIgnored()).toList()) { + if (cp.getClusterId() != null) { + clusteredByLocation.computeIfAbsent(cp.getClusterId(), _ -> new ArrayList<>()) + .add(cp.getPoint()); + } + } + + // Detect stay points + List stayPoints = detectStayPointsFromTrajectory(clusteredByLocation, detectionParams); + + // Create visits + List visits = stayPoints.stream() + .map(sp -> new Visit( + sp.getLongitude(), + sp.getLatitude(), + sp.getArrivalTime(), + sp.getDepartureTime(), + sp.getDurationSeconds(), + false + )) + .toList(); + + // Save visits + if (previewId == null) { + visits = visitJdbcService.bulkInsert(user, visits); + } else { + visits = previewVisitJdbcService.bulkInsert(user, previewId, visits); + } + + return new VisitDetectionResult(visits, windowStart, windowEnd); + } + + /** + * STEP 2: Visit Merging + * Merges nearby visits into ProcessedVisit entities with SignificantPlaces. + */ + private VisitMergingResult mergeVisits(User user, String previewId, String traceId, Instant initialStart, Instant initialEnd, List allVisits) { + + // Get merging parameters + DetectionParameter.VisitMerging mergeConfig; + if (previewId == null) { + mergeConfig = visitDetectionParametersService + .getCurrentConfiguration(user, initialStart) + .getVisitMerging(); + } else { + mergeConfig = previewVisitDetectionParametersJdbcService + .findCurrent(user, previewId) + .getVisitMerging(); + } + + // Expand search window for merging + Instant searchStart = initialStart.minus(mergeConfig.getSearchDurationInHours(), ChronoUnit.HOURS); + Instant searchEnd = initialEnd.plus(mergeConfig.getSearchDurationInHours(), ChronoUnit.HOURS); + + // Delete existing processed visits in range + List existingProcessedVisits; + if (previewId == null) { + existingProcessedVisits = processedVisitJdbcService + .findByUserAndStartTimeBeforeEqualAndEndTimeAfterEqual(user, searchEnd, searchStart); + processedVisitJdbcService.deleteAll(existingProcessedVisits); + } else { + existingProcessedVisits = previewProcessedVisitJdbcService + .findByUserAndStartTimeBeforeEqualAndEndTimeAfterEqual(user, previewId, searchEnd, searchStart); + previewProcessedVisitJdbcService.deleteAll(existingProcessedVisits); + } + + // Expand window based on deleted processed visits + if (!existingProcessedVisits.isEmpty()) { + if (existingProcessedVisits.getFirst().getStartTime().isBefore(searchStart)) { + searchStart = existingProcessedVisits.getFirst().getStartTime(); + } + if (existingProcessedVisits.getLast().getEndTime().isAfter(searchEnd)) { + searchEnd = existingProcessedVisits.getLast().getEndTime(); + } + } + + if (allVisits.isEmpty()) { + return new VisitMergingResult(List.of(), List.of(), searchStart, searchEnd); + } + + allVisits.sort(Comparator.comparing(Visit::getStartTime)); + // Merge visits chronologically + List processedVisits = mergeVisitsChronologically( + user, previewId, traceId, allVisits, mergeConfig); + + // Save processed visits + if (previewId == null) { + processedVisits = processedVisitJdbcService.bulkInsert(user, processedVisits); + } else { + processedVisits = previewProcessedVisitJdbcService.bulkInsert( + user, previewId, processedVisits); + } + + return new VisitMergingResult(allVisits, processedVisits, searchStart, searchEnd); + } + + /** + * STEP 3: Trip Detection + * Creates Trip entities between consecutive ProcessedVisits. + */ + private TripDetectionResult detectTrips(User user, String previewId, Instant searchStart, Instant searchEnd) { + + // Expand search for trip detection + searchStart = searchStart.minus(1, ChronoUnit.DAYS); + searchEnd = searchEnd.plus(1, ChronoUnit.DAYS); + + // Get all processed visits in range + List allProcessedVisits; + if (previewId == null) { + allProcessedVisits = processedVisitJdbcService.findByUserAndTimeOverlap( + user, searchStart, searchEnd); + } else { + allProcessedVisits = previewProcessedVisitJdbcService.findByUserAndTimeOverlap( + user, previewId, searchStart, searchEnd); + } + + if (allProcessedVisits.size() < 2) { + return new TripDetectionResult(List.of()); + } + + // Sort chronologically + allProcessedVisits.sort(Comparator.comparing(ProcessedVisit::getStartTime)); + + // Delete existing trips in range + if (previewId == null) { + List existingTrips = tripJdbcService.findByUserAndTimeOverlap( + user, searchStart, searchEnd); + tripJdbcService.deleteAll(existingTrips); + } else { + List existingTrips = previewTripJdbcService.findByUserAndTimeOverlap( + user, previewId, searchStart, searchEnd); + previewTripJdbcService.deleteAll(existingTrips); + } + + // Create trips between consecutive visits + List trips = new ArrayList<>(); + for (int i = 0; i < allProcessedVisits.size() - 1; i++) { + ProcessedVisit startVisit = allProcessedVisits.get(i); + ProcessedVisit endVisit = allProcessedVisits.get(i + 1); + + Trip trip = createTripBetweenVisits(user, previewId, startVisit, endVisit); + if (trip != null) { + trips.add(trip); + } + } + + // Save trips + if (previewId == null) { + trips = tripJdbcService.bulkInsert(user, trips); + } else { + trips = previewTripJdbcService.bulkInsert(user, previewId, trips); + } + + return new TripDetectionResult(trips); + } + + // ==================== Helper Methods ==================== + // Copy from existing services with minimal changes + + private List detectStayPointsFromTrajectory( + Map> points, + DetectionParameter.VisitDetection visitDetectionParameters) { + logger.debug("Starting cluster-based stay point detection with {} different spatial clusters.", points.size()); + + List> clusters = new ArrayList<>(); + + //split them up when time is x seconds between + for (List clusteredByLocation : points.values()) { + logger.debug("Start splitting up geospatial cluster with [{}] elements based on minimum time [{}]s between points", clusteredByLocation.size(), visitDetectionParameters.getMinimumStayTimeInSeconds()); + //first sort them by timestamp + clusteredByLocation.sort(Comparator.comparing(RawLocationPoint::getTimestamp)); + + List currentTimedCluster = new ArrayList<>(); + clusters.add(currentTimedCluster); + currentTimedCluster.add(clusteredByLocation.getFirst()); + + Instant currentTime = clusteredByLocation.getFirst().getTimestamp(); + + for (int i = 1; i < clusteredByLocation.size(); i++) { + RawLocationPoint next = clusteredByLocation.get(i); + if (Duration.between(currentTime, next.getTimestamp()).getSeconds() < visitDetectionParameters.getMaxMergeTimeBetweenSameStayPoints()) { + currentTimedCluster.add(next); + } else { + currentTimedCluster = new ArrayList<>(); + currentTimedCluster.add(next); + clusters.add(currentTimedCluster); + } + currentTime = next.getTimestamp(); + } + } + + logger.debug("Detected {} stay points after splitting them up.", clusters.size()); + //filter them by duration + List> filteredByMinimumDuration = clusters.stream() + .filter(c -> Duration.between(c.getFirst().getTimestamp(), c.getLast().getTimestamp()).toSeconds() > visitDetectionParameters.getMinimumStayTimeInSeconds()) + .toList(); + + logger.debug("Found {} valid clusters after duration filtering", filteredByMinimumDuration.size()); + + // Step 3: Convert valid clusters to stay points + return filteredByMinimumDuration.stream() + .map(this::createStayPoint) + .collect(Collectors.toList()); + } + + private List mergeVisitsChronologically( + User user, String previewId, String traceId, List visits, + DetectionParameter.VisitMerging mergeConfiguration) { + if (visits.isEmpty()) { + return new ArrayList<>(); + } + if (logger.isDebugEnabled()) { + logger.debug("Merging [{}] visits between [{}] and [{}]", visits.size(), visits.getFirst().getStartTime(), visits.getLast().getEndTime()); + } + List result = new ArrayList<>(); + + // Start with the first visit + Visit currentVisit = visits.getFirst(); + Instant currentStartTime = currentVisit.getStartTime(); + Instant currentEndTime = currentVisit.getEndTime(); + SignificantPlace currentPlace = findOrCreateSignificantPlace(user, previewId, currentVisit.getLatitude(), currentVisit.getLongitude(), mergeConfiguration, traceId); + + for (int i = 1; i < visits.size(); i++) { + Visit nextVisit = visits.get(i); + SignificantPlace nextPlace = findOrCreateSignificantPlace(user, previewId, nextVisit.getLatitude(), nextVisit.getLongitude(), mergeConfiguration, traceId); + + boolean samePlace = nextPlace.getId().equals(currentPlace.getId()); + boolean withinTimeThreshold = Duration.between(currentEndTime, nextVisit.getStartTime()).getSeconds() <= mergeConfiguration.getMaxMergeTimeBetweenSameVisits(); + + boolean shouldMergeWithNextVisit = samePlace && withinTimeThreshold; + + if (samePlace && !withinTimeThreshold) { + List pointsBetweenVisits; + if (previewId == null) { + pointsBetweenVisits = this.rawLocationPointJdbcService.findByUserAndTimestampBetweenOrderByTimestampAsc(user, currentEndTime, nextVisit.getStartTime()); + } else { + pointsBetweenVisits = this.previewRawLocationPointJdbcService.findByUserAndTimestampBetweenOrderByTimestampAsc(user, previewId, currentEndTime, nextVisit.getStartTime()); + } + if (pointsBetweenVisits.size() > 2) { + double travelledDistanceInMeters = GeoUtils.calculateTripDistance(pointsBetweenVisits); + shouldMergeWithNextVisit = travelledDistanceInMeters <= mergeConfiguration.getMinDistanceBetweenVisits(); + } else { + logger.debug("There are no points tracked between {} and {}. Will merge consecutive visits because they are on the same place", currentEndTime, nextVisit.getStartTime()); + shouldMergeWithNextVisit = true; + } + } + + if (shouldMergeWithNextVisit) { + currentEndTime = nextVisit.getEndTime().isAfter(currentEndTime) ? + nextVisit.getEndTime() : currentEndTime; + } else { + // Finalize the current merged visit + ProcessedVisit processedVisit = createProcessedVisit(currentPlace, currentStartTime, currentEndTime); + if (processedVisit != null) { + result.add(processedVisit); + + // This is the end time of the visit we just created. + Instant previousProcessedVisitEndTime = processedVisit.getEndTime(); + + // Start a new merged set, ensuring it does not start before the previous one ended. + currentPlace = nextPlace; + currentStartTime = nextVisit.getStartTime(); + currentEndTime = nextVisit.getEndTime(); + + // FIX: Adjust start time to prevent overlap. + if (currentStartTime.isBefore(previousProcessedVisitEndTime)) { + currentStartTime = previousProcessedVisitEndTime; + } + + // FIX: Ensure the end time is not before the (potentially adjusted) start time. + // This handles cases where a visit is completely enveloped by the previous one. + if (currentEndTime.isBefore(currentStartTime)) { + currentEndTime = currentStartTime; + } + } + } + } + + ProcessedVisit lastProcessedVisit = createProcessedVisit(currentPlace, currentStartTime, currentEndTime); + if (lastProcessedVisit != null) { + result.add(lastProcessedVisit); + } + return result; + } + + private ProcessedVisit createProcessedVisit(SignificantPlace place, Instant startTime, Instant endTime) { + if (endTime.isBefore(startTime)) { + logger.warn("Skipping zero or negative duration processed visit for place [{}] between [{}] and [{}]", place.getId(), startTime, endTime); + return null; // Indicate to skip + } + if (endTime.equals(startTime)) { + logger.warn("Skipping zero duration processed visit for place [{}] from [{} -> {}]", place.getId(), startTime, endTime); + return null; + } + logger.debug("Creating processed visit for place [{}] between [{}] and [{}]", place.getId(), startTime, endTime); + return new ProcessedVisit(place, startTime, endTime, endTime.getEpochSecond() - startTime.getEpochSecond()); + } + + private StayPoint createStayPoint(List clusterPoints) { + GeoPoint result = weightedCenter(clusterPoints); + + // Get the time range + Instant arrivalTime = clusterPoints.getFirst().getTimestamp(); + Instant departureTime = clusterPoints.getLast().getTimestamp(); + + logger.debug("Creating stay point at [{}] with arrival time [{}] and departure time [{}]", result, arrivalTime, departureTime); + return new StayPoint(result.latitude(), result.longitude(), arrivalTime, departureTime, clusterPoints); + } + + private GeoPoint weightedCenter(List clusterPoints) { + + long start = System.currentTimeMillis(); + + GeoPoint result; + // For small clusters, use the original algorithm + if (clusterPoints.size() <= 100) { + result = weightedCenterSimple(clusterPoints); + } else { + // For large clusters, use spatial partitioning for better performance + result = weightedCenterOptimized(clusterPoints); + } + logger.debug("Weighted center calculation took {}ms for [{}] number of points", System.currentTimeMillis() - start, clusterPoints.size()); + return result; + + } + + private GeoPoint weightedCenterSimple(List clusterPoints) { + RawLocationPoint bestPoint = null; + double maxDensityScore = 0; + + // For each point, calculate a density score based on nearby points and accuracy + for (RawLocationPoint candidate : clusterPoints) { + double densityScore = 0; + + for (RawLocationPoint neighbor : clusterPoints) { + if (candidate == neighbor) continue; + + double distance = GeoUtils.distanceInMeters(candidate, neighbor); + double accuracy = candidate.getAccuracyMeters() != null && candidate.getAccuracyMeters() > 0 + ? candidate.getAccuracyMeters() + : 50.0; // default accuracy if null + + // Points within accuracy radius contribute to density + // Closer points and better accuracy contribute more + if (distance <= accuracy * 2) { + double proximityWeight = Math.max(0, 1.0 - (distance / (accuracy * 2))); + double accuracyWeight = 1.0 / accuracy; + densityScore += proximityWeight * accuracyWeight; + } + } + + // Add self-contribution based on accuracy + densityScore += 1.0 / (candidate.getAccuracyMeters() != null && candidate.getAccuracyMeters() > 0 + ? candidate.getAccuracyMeters() + : 50.0); + + if (densityScore > maxDensityScore) { + maxDensityScore = densityScore; + bestPoint = candidate; + } + } + + // Fallback to first point if no best point found + if (bestPoint == null) { + bestPoint = clusterPoints.getFirst(); + } + + return new GeoPoint(bestPoint.getLatitude(), bestPoint.getLongitude()); + } + + private GeoPoint weightedCenterOptimized(List clusterPoints) { + // Sample a subset of points for density calculation to improve performance + // Use every nth point or random sampling for very large clusters + int sampleSize = Math.min(200, clusterPoints.size()); + List samplePoints = new ArrayList<>(); + + if (clusterPoints.size() <= sampleSize) { + samplePoints = clusterPoints; + } else { + // Take evenly distributed samples + int step = clusterPoints.size() / sampleSize; + for (int i = 0; i < clusterPoints.size(); i += step) { + samplePoints.add(clusterPoints.get(i)); + } + } + + // Use spatial grid approach to avoid distance calculations + // Create a grid based on the bounding box of all points + double minLat = clusterPoints.stream().mapToDouble(RawLocationPoint::getLatitude).min().orElse(0); + double minLon = clusterPoints.stream().mapToDouble(RawLocationPoint::getLongitude).min().orElse(0); + + // Grid cell size approximately 10 meters (rough approximation) + double cellSizeLat = 0.0001; // ~11 meters + double cellSizeLon = 0.0001; // varies by latitude but roughly 11 meters + + // Create grid map for fast neighbor lookup + Map> grid = new HashMap<>(); + for (RawLocationPoint point : clusterPoints) { + int gridLat = (int) ((point.getLatitude() - minLat) / cellSizeLat); + int gridLon = (int) ((point.getLongitude() - minLon) / cellSizeLon); + String gridKey = gridLat + "," + gridLon; + grid.computeIfAbsent(gridKey, _ -> new ArrayList<>()).add(point); + } + + RawLocationPoint bestPoint = null; + double maxDensityScore = 0; + + // Calculate density scores for sample points using grid lookup + for (RawLocationPoint candidate : samplePoints) { + double accuracy = candidate.getAccuracyMeters() != null && candidate.getAccuracyMeters() > 0 + ? candidate.getAccuracyMeters() + : 50.0; + + // Calculate grid coordinates for candidate + int candidateGridLat = (int) ((candidate.getLatitude() - minLat) / cellSizeLat); + int candidateGridLon = (int) ((candidate.getLongitude() - minLon) / cellSizeLon); + + // Search radius in grid cells (conservative estimate) + int searchRadiusInCells = Math.max(1, (int) (accuracy / 100000)); // rough conversion + + double densityScore = 0; + + // Check neighboring grid cells + for (int latOffset = -searchRadiusInCells; latOffset <= searchRadiusInCells; latOffset++) { + for (int lonOffset = -searchRadiusInCells; lonOffset <= searchRadiusInCells; lonOffset++) { + String neighborKey = (candidateGridLat + latOffset) + "," + (candidateGridLon + lonOffset); + List neighbors = grid.get(neighborKey); + + if (neighbors != null) { + for (RawLocationPoint neighbor : neighbors) { + if (candidate != neighbor) { + // Simple proximity weight based on grid distance + double gridDistance = Math.sqrt(latOffset * latOffset + lonOffset * lonOffset); + double proximityWeight = Math.max(0, 1.0 - (gridDistance / searchRadiusInCells)); + densityScore += proximityWeight; + } + } + } + } + } + + // Combine density with accuracy weight + double accuracyWeight = 1.0 / accuracy; + densityScore = (densityScore * accuracyWeight) + accuracyWeight; + + if (densityScore > maxDensityScore) { + maxDensityScore = densityScore; + bestPoint = candidate; + } + } + + // Fallback to first point if no best point found + if (bestPoint == null) { + bestPoint = clusterPoints.getFirst(); + } + + return new GeoPoint(bestPoint.getLatitude(), bestPoint.getLongitude()); + } + + private Trip createTripBetweenVisits(User user, String previewId, + ProcessedVisit startVisit, ProcessedVisit endVisit) { + // Trip starts when the first visit ends + Instant tripStartTime = startVisit.getEndTime(); + + // Trip ends when the second visit starts + Instant tripEndTime = endVisit.getStartTime(); + + if (previewId != null) { + if (this.previewProcessedVisitJdbcService.findById(startVisit.getId()).isEmpty() || this.previewProcessedVisitJdbcService.findById(endVisit.getId()).isEmpty()) { + logger.debug("One of the following preview visits [{},{}] where already deleted. Will skip trip creation.", startVisit.getId(), endVisit.getId()); + return null; + } + } else { + if (this.processedVisitJdbcService.findById(startVisit.getId()).isEmpty() || this.processedVisitJdbcService.findById(endVisit.getId()).isEmpty()) { + logger.debug("One of the following visits [{},{}] where already deleted. Will skip trip creation.", startVisit.getId(), endVisit.getId()); + return null; + } + } + // If end time is before or equal to start time, this is not a valid trip + if (tripEndTime.isBefore(tripStartTime) || tripEndTime.equals(tripStartTime)) { + logger.warn("Invalid trip time range detected for user {}: {} to {}", + user.getUsername(), tripStartTime, tripEndTime); + return null; + } + + + if (previewId == null) { + // Check if a trip already exists with the same start and end times + if (tripJdbcService.existsByUserAndStartTimeAndEndTime(user, tripStartTime, tripEndTime)) { + logger.debug("Trip already exists for user {} from {} to {}", + user.getUsername(), tripStartTime, tripEndTime); + return null; + } + } + + // Get location points between the two visits + List tripPoints; + if (previewId == null) { + tripPoints = rawLocationPointJdbcService.findByUserAndTimestampBetweenOrderByTimestampAsc(user, tripStartTime, tripEndTime); + } else { + tripPoints = previewRawLocationPointJdbcService.findByUserAndTimestampBetweenOrderByTimestampAsc(user, previewId, tripStartTime, tripEndTime); + } + double estimatedDistanceInMeters = calculateDistanceBetweenPlaces(startVisit.getPlace(), endVisit.getPlace()); + double travelledDistanceMeters = GeoUtils.calculateTripDistance(tripPoints); + // Create a new trip + TransportMode transportMode = this.transportModeService.inferTransportMode(user, tripPoints, tripStartTime, tripEndTime); + Trip trip = new Trip( + tripStartTime, + tripEndTime, + tripEndTime.getEpochSecond() - tripStartTime.getEpochSecond(), + estimatedDistanceInMeters, + travelledDistanceMeters, + transportMode, + startVisit, + endVisit + ); + logger.debug("Created trip from {} to {}: travelled distance={}m, mode={}", + Optional.ofNullable(startVisit.getPlace().getName()).orElse("Unknown Name"), + Optional.ofNullable(endVisit.getPlace().getName()).orElse("Unknown Name"), + Math.round(travelledDistanceMeters), + transportMode); + + // Save and return the trip + return trip; + } + + private List findNearbyPlaces(User user, String previewId, double latitude, double longitude, DetectionParameter.VisitMerging mergeConfiguration) { + // Create a point geometry + Point point = geometryFactory.createPoint(new Coordinate(longitude, latitude)); + // Find places within the merge distance + if (previewId == null) { + return significantPlaceJdbcService.findNearbyPlaces(user.getId(), point, GeoUtils.metersToDegreesAtPosition(mergeConfiguration.getMinDistanceBetweenVisits() / 2.0, latitude)); + } else { + return previewSignificantPlaceJdbcService.findNearbyPlaces(user.getId(), point, GeoUtils.metersToDegreesAtPosition(mergeConfiguration.getMinDistanceBetweenVisits() / 2.0, latitude), previewId); + } + } + + private SignificantPlace findOrCreateSignificantPlace(User user, String previewId, + double latitude, double longitude, + DetectionParameter.VisitMerging mergeConfig, + String traceId) { + List nearbyPlaces = findNearbyPlaces(user, previewId, latitude, longitude, mergeConfig); + return nearbyPlaces.isEmpty() ? createSignificantPlace(user, latitude, longitude, previewId, traceId) : findClosestPlace(latitude, longitude, nearbyPlaces); + } + + + private SignificantPlace createSignificantPlace(User user, double latitude, double longitude, String previewId, String traceId) { + SignificantPlace significantPlace = SignificantPlace.create(latitude, longitude); + Optional timezone = this.timezoneService.getTimezone(significantPlace); + if (timezone.isPresent()) { + significantPlace = significantPlace.withTimezone(timezone.get()); + } + // Check for override + GeoPoint point = new GeoPoint(significantPlace.getLatitudeCentroid(), significantPlace.getLongitudeCentroid()); + Optional override = significantPlaceOverrideJdbcService.findByUserAndPoint(user, point); + if (override.isPresent()) { + logger.info("Found override for user [{}] and location [{}], using override information: {}", user.getUsername(), point, override.get()); + significantPlace = significantPlace + .withName(override.get().name()) + .withType(override.get().category()) + .withTimezone(override.get().timezone()); + } + significantPlace = previewId == null ? this.significantPlaceJdbcService.create(user, significantPlace) : this.previewSignificantPlaceJdbcService.create(user, previewId, significantPlace); + publishSignificantPlaceCreatedEvent(user, significantPlace, previewId, traceId); + return significantPlace; + } + + private SignificantPlace findClosestPlace(double latitude, double longitude, List places) { + + Comparator distanceComparator = Comparator.comparingDouble(place -> + GeoUtils.distanceInMeters( + latitude, longitude, + place.getLatitudeCentroid(), place.getLongitudeCentroid())); + return places.stream() + .min(distanceComparator.thenComparing(SignificantPlace::getId)) + .orElseThrow(() -> new IllegalStateException("No places found")); + } + + private double calculateDistanceBetweenPlaces(SignificantPlace place1, SignificantPlace place2) { + return GeoUtils.distanceInMeters( + place1.getLatitudeCentroid(), place1.getLongitudeCentroid(), + place2.getLatitudeCentroid(), place2.getLongitudeCentroid()); + } + + private void publishSignificantPlaceCreatedEvent(User user, SignificantPlace place, String previewId, String traceId) { + SignificantPlaceCreatedEvent event = new SignificantPlaceCreatedEvent( + user.getUsername(), + previewId, + place.getId(), + place.getLatitudeCentroid(), + place.getLongitudeCentroid(), + traceId + ); + rabbitTemplate.convertAndSend(RabbitMQConfig.EXCHANGE_NAME, RabbitMQConfig.SIGNIFICANT_PLACE_ROUTING_KEY, event); + logger.info("Published SignificantPlaceCreatedEvent for place ID: {}", place.getId()); + } + + // ==================== Result Classes ==================== + + private record VisitDetectionResult(List visits, Instant searchStart, Instant searchEnd) { + } + + private record VisitMergingResult(List inputVisits, List processedVisits, + Instant searchStart, Instant searchEnd) { + } + + private record TripDetectionResult(List trips) { + } +} \ No newline at end of file diff --git a/src/main/java/com/dedicatedcode/reitti/service/processing/VisitDetectionService.java b/src/main/java/com/dedicatedcode/reitti/service/processing/VisitDetectionService.java deleted file mode 100644 index 3471468c..00000000 --- a/src/main/java/com/dedicatedcode/reitti/service/processing/VisitDetectionService.java +++ /dev/null @@ -1,369 +0,0 @@ -package com.dedicatedcode.reitti.service.processing; - -import com.dedicatedcode.reitti.config.RabbitMQConfig; -import com.dedicatedcode.reitti.event.LocationProcessEvent; -import com.dedicatedcode.reitti.event.VisitUpdatedEvent; -import com.dedicatedcode.reitti.model.ClusteredPoint; -import com.dedicatedcode.reitti.model.geo.*; -import com.dedicatedcode.reitti.model.processing.DetectionParameter; -import com.dedicatedcode.reitti.model.security.User; -import com.dedicatedcode.reitti.repository.*; -import com.dedicatedcode.reitti.service.VisitDetectionParametersService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.time.Duration; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; - -@Service -public class VisitDetectionService { - private static final Logger logger = LoggerFactory.getLogger(VisitDetectionService.class); - - private final UserJdbcService userJdbcService; - private final RawLocationPointJdbcService rawLocationPointJdbcService; - private final VisitDetectionParametersService visitDetectionParametersService; - private final PreviewRawLocationPointJdbcService previewRawLocationPointJdbcService; - private final VisitJdbcService visitJdbcService; - private final PreviewVisitJdbcService previewVisitJdbcService; - - private final RabbitTemplate rabbitTemplate; - private final ConcurrentHashMap userLocks = new ConcurrentHashMap<>(); - - @Autowired - public VisitDetectionService( - RawLocationPointJdbcService rawLocationPointJdbcService, - PreviewRawLocationPointJdbcService previewRawLocationPointJdbcService, - VisitDetectionParametersService visitDetectionParametersService, - UserJdbcService userJdbcService, - VisitJdbcService visitJdbcService, - PreviewVisitJdbcService previewVisitJdbcService, - RabbitTemplate rabbitTemplate) { - this.rawLocationPointJdbcService = rawLocationPointJdbcService; - this.visitDetectionParametersService = visitDetectionParametersService; - this.userJdbcService = userJdbcService; - this.previewRawLocationPointJdbcService = previewRawLocationPointJdbcService; - this.visitJdbcService = visitJdbcService; - this.previewVisitJdbcService = previewVisitJdbcService; - this.rabbitTemplate = rabbitTemplate; - } - - public void detectStayPoints(LocationProcessEvent incoming) { - String username = incoming.getUsername(); - ReentrantLock userLock = userLocks.computeIfAbsent(username, _ -> new ReentrantLock()); - - userLock.lock(); - try { - logger.debug("Detecting stay points for user {} from {} to {}. Mode: {}", username, incoming.getEarliest(), incoming.getLatest(), incoming.getPreviewId() == null ? "live" : "preview"); - User user = userJdbcService.findByUsername(username).orElseThrow(); - // We extend the search window slightly to catch visits spanning midnight - Instant windowStart = incoming.getEarliest().minus(5, ChronoUnit.MINUTES); - // Get points from 1 day after the latest new point - Instant windowEnd = incoming.getLatest().plus(5, ChronoUnit.MINUTES); - - DetectionParameter.VisitDetection detectionParameters; - if (incoming.getPreviewId() == null) { - detectionParameters = this.visitDetectionParametersService.getCurrentConfiguration(user, windowStart).getVisitDetection(); - } else { - detectionParameters = this.visitDetectionParametersService.getCurrentConfiguration(user, incoming.getPreviewId()).getVisitDetection(); - - } - List affectedVisits; - if (incoming.getPreviewId() == null) { - affectedVisits = this.visitJdbcService.findByUserAndTimeAfterAndStartTimeBefore(user, windowStart, windowEnd); - } else { - affectedVisits = this.previewVisitJdbcService.findByUserAndTimeAfterAndStartTimeBefore(user, incoming.getPreviewId(), windowStart, windowEnd); - } - if (logger.isDebugEnabled()) { - logger.debug("Found [{}] visits which touch the timerange from [{}] to [{}]", affectedVisits.size(), windowStart, windowEnd); - affectedVisits.forEach(visit -> logger.debug("Visit [{}] from [{}] to [{}] at [{},{}]", visit.getId(), visit.getStartTime(), visit.getEndTime(), visit.getLongitude(), visit.getLatitude())); - - } - try { - if (incoming.getPreviewId() == null) { - this.visitJdbcService.delete(affectedVisits); - } else { - this.previewVisitJdbcService.delete(affectedVisits); - } - logger.debug("Deleted [{}] visits with ids [{}]", affectedVisits.size(), affectedVisits.stream().map(Visit::getId).map(Object::toString).collect(Collectors.joining())); - } catch (OptimisticLockException e) { - logger.error("Optimistic lock exception", e); - throw new RuntimeException(e); - } - - if (!affectedVisits.isEmpty()) { - if (affectedVisits.getFirst().getStartTime().isBefore(windowStart)) { - windowStart = affectedVisits.getFirst().getStartTime(); - } - - if (affectedVisits.getLast().getEndTime().isAfter(windowEnd)) { - windowEnd = affectedVisits.getLast().getEndTime(); - } - } - logger.debug("Searching for points in the timerange from [{}] to [{}]", windowStart, windowEnd); - - double baseLatitude = affectedVisits.isEmpty() ? 50 : affectedVisits.getFirst().getLatitude(); - double[] metersAsDegrees = GeoUtils.metersToDegreesAtPosition(detectionParameters.getSearchDistanceInMeters(), baseLatitude); - List clusteredPointsInTimeRangeForUser; - if (incoming.getPreviewId() == null) { - clusteredPointsInTimeRangeForUser = this.rawLocationPointJdbcService.findClusteredPointsInTimeRangeForUser(user, windowStart, windowEnd, detectionParameters.getMinimumAdjacentPoints(), metersAsDegrees[0]); - } else { - clusteredPointsInTimeRangeForUser = this.previewRawLocationPointJdbcService.findClusteredPointsInTimeRangeForUser(user, incoming.getPreviewId(), windowStart, windowEnd, detectionParameters.getMinimumAdjacentPoints(), metersAsDegrees[0]); - } - - Map> clusteredByLocation = new HashMap<>(); - for (ClusteredPoint clusteredPoint : clusteredPointsInTimeRangeForUser) { - if (clusteredPoint.getClusterId() != null) { - clusteredByLocation.computeIfAbsent(clusteredPoint.getClusterId(), _ -> new ArrayList<>()).add(clusteredPoint.getPoint()); - } - } - - logger.debug("Found {} point clusters in the processing window from [{}] to [{}]", clusteredByLocation.size(), windowStart, windowEnd); - - // Apply the stay point detection algorithm - List stayPoints = detectStayPointsFromTrajectory(clusteredByLocation, detectionParameters); - - logger.info("Detected {} stay points for user {}", stayPoints.size(), user.getUsername()); - - List createdVisits = new ArrayList<>(); - - for (StayPoint stayPoint : stayPoints) { - Visit visit = createVisit(stayPoint.getLongitude(), stayPoint.getLatitude(), stayPoint); - logger.debug("Creating new visit: {}", visit); - createdVisits.add(visit); - } - - List createdIds; - if (incoming.getPreviewId() == null) { - createdIds = visitJdbcService.bulkInsert(user, createdVisits).stream().map(Visit::getId).toList(); - } else { - createdIds = previewVisitJdbcService.bulkInsert(user, incoming.getPreviewId(), createdVisits).stream().map(Visit::getId).toList(); - } - rabbitTemplate.convertAndSend(RabbitMQConfig.EXCHANGE_NAME, RabbitMQConfig.MERGE_VISIT_ROUTING_KEY, new VisitUpdatedEvent(user.getUsername(), createdIds, incoming.getPreviewId())); - } finally { - userLock.unlock(); - } - } - - - private List detectStayPointsFromTrajectory(Map> points, DetectionParameter.VisitDetection visitDetectionParameters) { - logger.debug("Starting cluster-based stay point detection with {} different spatial clusters.", points.size()); - - List> clusters = new ArrayList<>(); - - //split them up when time is x seconds between - for (List clusteredByLocation : points.values()) { - logger.debug("Start splitting up geospatial cluster with [{}] elements based on minimum time [{}]s between points", clusteredByLocation.size(), visitDetectionParameters.getMinimumStayTimeInSeconds()); - //first sort them by timestamp - clusteredByLocation.sort(Comparator.comparing(RawLocationPoint::getTimestamp)); - - List currentTimedCluster = new ArrayList<>(); - clusters.add(currentTimedCluster); - currentTimedCluster.add(clusteredByLocation.getFirst()); - - Instant currentTime = clusteredByLocation.getFirst().getTimestamp(); - - for (int i = 1; i < clusteredByLocation.size(); i++) { - RawLocationPoint next = clusteredByLocation.get(i); - if (Duration.between(currentTime, next.getTimestamp()).getSeconds() < visitDetectionParameters.getMaxMergeTimeBetweenSameStayPoints()) { - currentTimedCluster.add(next); - } else { - currentTimedCluster = new ArrayList<>(); - currentTimedCluster.add(next); - clusters.add(currentTimedCluster); - } - currentTime = next.getTimestamp(); - } - } - - logger.debug("Detected {} stay points after splitting them up.", clusters.size()); - //filter them by duration - List> filteredByMinimumDuration = clusters.stream() - .filter(c -> Duration.between(c.getFirst().getTimestamp(), c.getLast().getTimestamp()).toSeconds() > visitDetectionParameters.getMinimumStayTimeInSeconds()) - .toList(); - - logger.debug("Found {} valid clusters after duration filtering", filteredByMinimumDuration.size()); - - // Step 3: Convert valid clusters to stay points - return filteredByMinimumDuration.stream() - .map(this::createStayPoint) - .collect(Collectors.toList()); - } - - private StayPoint createStayPoint(List clusterPoints) { - GeoPoint result = weightedCenter(clusterPoints); - - // Get the time range - Instant arrivalTime = clusterPoints.getFirst().getTimestamp(); - Instant departureTime = clusterPoints.getLast().getTimestamp(); - - return new StayPoint(result.latitude(), result.longitude(), arrivalTime, departureTime, clusterPoints); - } - - private GeoPoint weightedCenter(List clusterPoints) { - // Find the most likely actual location by identifying the point with highest local density - // and snapping to the nearest actual measurement point - - long start = System.currentTimeMillis(); - - GeoPoint result; - // For small clusters, use the original algorithm - if (clusterPoints.size() <= 100) { - result = weightedCenterSimple(clusterPoints); - } else { - // For large clusters, use spatial partitioning for better performance - result = weightedCenterOptimized(clusterPoints); - } - logger.debug("Weighted center calculation took {}ms for [{}] number of points", System.currentTimeMillis() - start, clusterPoints.size()); - return result; - - } - - private GeoPoint weightedCenterSimple(List clusterPoints) { - RawLocationPoint bestPoint = null; - double maxDensityScore = 0; - - // For each point, calculate a density score based on nearby points and accuracy - for (RawLocationPoint candidate : clusterPoints) { - double densityScore = 0; - - for (RawLocationPoint neighbor : clusterPoints) { - if (candidate == neighbor) continue; - - double distance = GeoUtils.distanceInMeters(candidate, neighbor); - double accuracy = candidate.getAccuracyMeters() != null && candidate.getAccuracyMeters() > 0 - ? candidate.getAccuracyMeters() - : 50.0; // default accuracy if null - - // Points within accuracy radius contribute to density - // Closer points and better accuracy contribute more - if (distance <= accuracy * 2) { - double proximityWeight = Math.max(0, 1.0 - (distance / (accuracy * 2))); - double accuracyWeight = 1.0 / accuracy; - densityScore += proximityWeight * accuracyWeight; - } - } - - // Add self-contribution based on accuracy - densityScore += 1.0 / (candidate.getAccuracyMeters() != null && candidate.getAccuracyMeters() > 0 - ? candidate.getAccuracyMeters() - : 50.0); - - if (densityScore > maxDensityScore) { - maxDensityScore = densityScore; - bestPoint = candidate; - } - } - - // Fallback to first point if no best point found - if (bestPoint == null) { - bestPoint = clusterPoints.getFirst(); - } - - return new GeoPoint(bestPoint.getLatitude(), bestPoint.getLongitude()); - } - - private GeoPoint weightedCenterOptimized(List clusterPoints) { - // Sample a subset of points for density calculation to improve performance - // Use every nth point or random sampling for very large clusters - int sampleSize = Math.min(200, clusterPoints.size()); - List samplePoints = new ArrayList<>(); - - if (clusterPoints.size() <= sampleSize) { - samplePoints = clusterPoints; - } else { - // Take evenly distributed samples - int step = clusterPoints.size() / sampleSize; - for (int i = 0; i < clusterPoints.size(); i += step) { - samplePoints.add(clusterPoints.get(i)); - } - } - - // Use spatial grid approach to avoid distance calculations - // Create a grid based on the bounding box of all points - double minLat = clusterPoints.stream().mapToDouble(RawLocationPoint::getLatitude).min().orElse(0); - double maxLat = clusterPoints.stream().mapToDouble(RawLocationPoint::getLatitude).max().orElse(0); - double minLon = clusterPoints.stream().mapToDouble(RawLocationPoint::getLongitude).min().orElse(0); - double maxLon = clusterPoints.stream().mapToDouble(RawLocationPoint::getLongitude).max().orElse(0); - - // Grid cell size approximately 10 meters (rough approximation) - double cellSizeLat = 0.0001; // ~11 meters - double cellSizeLon = 0.0001; // varies by latitude but roughly 11 meters - - // Create grid map for fast neighbor lookup - Map> grid = new HashMap<>(); - for (RawLocationPoint point : clusterPoints) { - int gridLat = (int) ((point.getLatitude() - minLat) / cellSizeLat); - int gridLon = (int) ((point.getLongitude() - minLon) / cellSizeLon); - String gridKey = gridLat + "," + gridLon; - grid.computeIfAbsent(gridKey, k -> new ArrayList<>()).add(point); - } - - RawLocationPoint bestPoint = null; - double maxDensityScore = 0; - - // Calculate density scores for sample points using grid lookup - for (RawLocationPoint candidate : samplePoints) { - double accuracy = candidate.getAccuracyMeters() != null && candidate.getAccuracyMeters() > 0 - ? candidate.getAccuracyMeters() - : 50.0; - - // Calculate grid coordinates for candidate - int candidateGridLat = (int) ((candidate.getLatitude() - minLat) / cellSizeLat); - int candidateGridLon = (int) ((candidate.getLongitude() - minLon) / cellSizeLon); - - // Search radius in grid cells (conservative estimate) - int searchRadiusInCells = Math.max(1, (int) (accuracy / 100000)); // rough conversion - - double densityScore = 0; - int nearbyCount = 0; - - // Check neighboring grid cells - for (int latOffset = -searchRadiusInCells; latOffset <= searchRadiusInCells; latOffset++) { - for (int lonOffset = -searchRadiusInCells; lonOffset <= searchRadiusInCells; lonOffset++) { - String neighborKey = (candidateGridLat + latOffset) + "," + (candidateGridLon + lonOffset); - List neighbors = grid.get(neighborKey); - - if (neighbors != null) { - for (RawLocationPoint neighbor : neighbors) { - if (candidate != neighbor) { - nearbyCount++; - // Simple proximity weight based on grid distance - double gridDistance = Math.sqrt(latOffset * latOffset + lonOffset * lonOffset); - double proximityWeight = Math.max(0, 1.0 - (gridDistance / searchRadiusInCells)); - densityScore += proximityWeight; - } - } - } - } - } - - // Combine density with accuracy weight - double accuracyWeight = 1.0 / accuracy; - densityScore = (densityScore * accuracyWeight) + accuracyWeight; - - if (densityScore > maxDensityScore) { - maxDensityScore = densityScore; - bestPoint = candidate; - } - } - - // Fallback to first point if no best point found - if (bestPoint == null) { - bestPoint = clusterPoints.getFirst(); - } - - return new GeoPoint(bestPoint.getLatitude(), bestPoint.getLongitude()); - } - - private Visit createVisit(Double longitude, Double latitude, StayPoint stayPoint) { - return new Visit(longitude, latitude, stayPoint.getArrivalTime(), stayPoint.getDepartureTime(), stayPoint.getDurationSeconds(), false); - } -} diff --git a/src/main/java/com/dedicatedcode/reitti/service/processing/VisitMergingService.java b/src/main/java/com/dedicatedcode/reitti/service/processing/VisitMergingService.java deleted file mode 100644 index b0cd5331..00000000 --- a/src/main/java/com/dedicatedcode/reitti/service/processing/VisitMergingService.java +++ /dev/null @@ -1,310 +0,0 @@ -package com.dedicatedcode.reitti.service.processing; - -import com.dedicatedcode.reitti.config.RabbitMQConfig; -import com.dedicatedcode.reitti.event.ProcessedVisitCreatedEvent; -import com.dedicatedcode.reitti.event.SignificantPlaceCreatedEvent; -import com.dedicatedcode.reitti.event.VisitUpdatedEvent; -import com.dedicatedcode.reitti.model.PlaceInformationOverride; -import com.dedicatedcode.reitti.model.geo.*; -import com.dedicatedcode.reitti.model.processing.DetectionParameter; -import com.dedicatedcode.reitti.model.security.User; -import com.dedicatedcode.reitti.repository.*; -import com.dedicatedcode.reitti.service.GeoLocationTimezoneService; -import com.dedicatedcode.reitti.service.UserNotificationService; -import com.dedicatedcode.reitti.service.VisitDetectionParametersService; -import org.locationtech.jts.geom.Coordinate; -import org.locationtech.jts.geom.GeometryFactory; -import org.locationtech.jts.geom.Point; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.time.Duration; -import java.time.Instant; -import java.time.ZoneId; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; - -@Service -public class VisitMergingService { - - private static final Logger logger = LoggerFactory.getLogger(VisitMergingService.class); - - private final VisitJdbcService visitJdbcService; - private final PreviewVisitJdbcService previewVisitJdbcService; - private final ProcessedVisitJdbcService processedVisitJdbcService; - private final PreviewProcessedVisitJdbcService previewProcessedVisitJdbcService; - private final UserJdbcService userJdbcService; - private final SignificantPlaceJdbcService significantPlaceJdbcService; - private final PreviewSignificantPlaceJdbcService previewSignificantPlaceJdbcService; - private final RawLocationPointJdbcService rawLocationPointJdbcService; - private final PreviewRawLocationPointJdbcService previewRawLocationPointJdbcService; - private final SignificantPlaceOverrideJdbcService significantPlaceOverrideJdbcService; - private final GeometryFactory geometryFactory; - private final RabbitTemplate rabbitTemplate; - private final UserNotificationService userNotificationService; - private final GeoLocationTimezoneService timezoneService; - private final VisitDetectionParametersService visitDetectionParametersService; - - @Autowired - public VisitMergingService(VisitJdbcService visitJdbcService, - PreviewVisitJdbcService previewVisitJdbcService, - ProcessedVisitJdbcService processedVisitJdbcService, - PreviewProcessedVisitJdbcService previewProcessedVisitJdbcService, - UserJdbcService userJdbcService, - RabbitTemplate rabbitTemplate, - SignificantPlaceJdbcService significantPlaceJdbcService, - PreviewSignificantPlaceJdbcService previewSignificantPlaceJdbcService, - RawLocationPointJdbcService rawLocationPointJdbcService, - PreviewRawLocationPointJdbcService previewRawLocationPointJdbcService, - SignificantPlaceOverrideJdbcService significantPlaceOverrideJdbcService, - GeometryFactory geometryFactory, - UserNotificationService userNotificationService, - GeoLocationTimezoneService timezoneService, - VisitDetectionParametersService visitDetectionParametersService) { - this.visitJdbcService = visitJdbcService; - this.previewVisitJdbcService = previewVisitJdbcService; - this.processedVisitJdbcService = processedVisitJdbcService; - this.previewProcessedVisitJdbcService = previewProcessedVisitJdbcService; - this.userJdbcService = userJdbcService; - this.rabbitTemplate = rabbitTemplate; - this.significantPlaceJdbcService = significantPlaceJdbcService; - this.previewSignificantPlaceJdbcService = previewSignificantPlaceJdbcService; - this.rawLocationPointJdbcService = rawLocationPointJdbcService; - this.previewRawLocationPointJdbcService = previewRawLocationPointJdbcService; - this.significantPlaceOverrideJdbcService = significantPlaceOverrideJdbcService; - this.geometryFactory = geometryFactory; - this.userNotificationService = userNotificationService; - this.timezoneService = timezoneService; - this.visitDetectionParametersService = visitDetectionParametersService; - } - - public void visitUpdated(VisitUpdatedEvent event) { - String username = event.getUsername(); - handleEvent(username, event.getVisitIds(), event.getPreviewId()); - } - - private void handleEvent(String username, List visitIds, String previewId) { - Optional user = userJdbcService.findByUsername(username); - if (user.isEmpty()) { - logger.warn("User not found for userName: {}", username); - return; - } - List visits = previewId == null ? this.visitJdbcService.findAllByIds(visitIds) : this.previewVisitJdbcService.findAllByIds(visitIds); - if (visits.isEmpty()) { - logger.debug("Visit not found for visitId: [{}]", visitIds); - return; - } - - Instant firstVisitTime = visits.stream().map(Visit::getStartTime).min(Comparator.naturalOrder()).orElseThrow(); - DetectionParameter.VisitMerging mergeConfiguration; - if (previewId == null) { - mergeConfiguration = this.visitDetectionParametersService.getCurrentConfiguration(user.get(), firstVisitTime).getVisitMerging(); - } else{ - mergeConfiguration = this.visitDetectionParametersService.getCurrentConfiguration(user.get(), previewId).getVisitMerging(); - } - Instant searchStart = visits.stream().min(Comparator.comparing(Visit::getStartTime)).map(Visit::getStartTime).map(instant -> instant.minus(mergeConfiguration.getSearchDurationInHours(), ChronoUnit.HOURS)).orElseThrow(); - Instant searchEnd = visits.stream().max(Comparator.comparing(Visit::getEndTime)).map(Visit::getEndTime).map(instant -> instant.plus(mergeConfiguration.getSearchDurationInHours(), ChronoUnit.HOURS)).orElseThrow(); - - processAndMergeVisits(user.get(), previewId, searchStart, searchEnd, mergeConfiguration); - } - - private void processAndMergeVisits(User user, String previewId, Instant searchStart, Instant searchEnd, DetectionParameter.VisitMerging mergeConfiguration) { - logger.info("Processing and merging visits for user: [{}] between [{}] and [{}]", user.getUsername(), searchStart, searchEnd); - List allProcessedVisitsInRange; - if (previewId == null) { - - allProcessedVisitsInRange = this.processedVisitJdbcService.findByUserAndStartTimeBeforeEqualAndEndTimeAfterEqual(user, searchEnd, searchStart); - logger.debug("found [{}] processed visits in range [{}] to [{}]", allProcessedVisitsInRange.size(), searchStart, searchEnd); - this.processedVisitJdbcService.deleteAll(allProcessedVisitsInRange); - } else { - allProcessedVisitsInRange = this.previewProcessedVisitJdbcService.findByUserAndStartTimeBeforeEqualAndEndTimeAfterEqual(user, previewId, searchEnd, searchStart); - logger.debug("found [{}] processed preview visits in range [{}] to [{}]", allProcessedVisitsInRange.size(), searchStart, searchEnd); - this.previewProcessedVisitJdbcService.deleteAll(allProcessedVisitsInRange); - - } - if (!allProcessedVisitsInRange.isEmpty()) { - if (allProcessedVisitsInRange.getFirst().getStartTime().isBefore(searchStart)) { - searchStart = allProcessedVisitsInRange.getFirst().getStartTime(); - } - if (allProcessedVisitsInRange.getLast().getEndTime().isAfter(searchEnd)) { - searchEnd = allProcessedVisitsInRange.getLast().getEndTime(); - } - } - - logger.debug("After finding [{}] existing processed visits, expanding search range for Visits between [{}] and [{}]", allProcessedVisitsInRange.size(), searchStart, searchEnd); - - List allVisits; - if (previewId == null) { - allVisits = this.visitJdbcService.findByUserAndTimeAfterAndStartTimeBefore(user, searchStart, searchEnd); - } else { - allVisits = this.previewVisitJdbcService.findByUserAndTimeAfterAndStartTimeBefore(user, previewId, searchStart, searchEnd); - } - if (allVisits.isEmpty()) { - logger.info("No visits found for user: {}", user.getUsername()); - return; - } - - // Process all visits chronologically to avoid overlaps - List processedVisits = mergeVisitsChronologically(user, previewId, allVisits, mergeConfiguration); - - if (previewId == null) { - processedVisitJdbcService.bulkInsert(user, processedVisits) - .stream() - .sorted(Comparator.comparing(ProcessedVisit::getStartTime)) - .forEach(processedVisit -> this.rabbitTemplate.convertAndSend(RabbitMQConfig.EXCHANGE_NAME, RabbitMQConfig.DETECT_TRIP_ROUTING_KEY, new ProcessedVisitCreatedEvent(user.getUsername(), processedVisit.getId(), null))); - } else { - previewProcessedVisitJdbcService.bulkInsert(user, previewId, processedVisits) - .stream() - .sorted(Comparator.comparing(ProcessedVisit::getStartTime)) - .forEach(processedVisit -> this.rabbitTemplate.convertAndSend(RabbitMQConfig.EXCHANGE_NAME, RabbitMQConfig.DETECT_TRIP_ROUTING_KEY, new ProcessedVisitCreatedEvent(user.getUsername(), processedVisit.getId(), previewId))); - } - logger.debug("Processed [{}] visits into [{}] merged visits for user: [{}]", - allVisits.size(), processedVisits.size(), user.getUsername()); - if (previewId == null) { - this.userNotificationService.newVisits(user, processedVisits); - } - } - - - private List mergeVisitsChronologically(User user, String previewId, List visits, DetectionParameter.VisitMerging mergeConfiguration) { - if (logger.isDebugEnabled()) { - logger.debug("Merging [{}] visits between [{}] and [{}]", visits.size(), visits.getFirst().getStartTime(), visits.getLast().getEndTime()); - } - List result = new ArrayList<>(); - - if (visits.isEmpty()) { - return result; - } - - // Start with the first visit - Visit currentVisit = visits.getFirst(); - Instant currentStartTime = currentVisit.getStartTime(); - Instant currentEndTime = currentVisit.getEndTime(); - - // Find or create a place for the first visit - List nearbyPlaces = findNearbyPlaces(user, previewId, currentVisit.getLatitude(), currentVisit.getLongitude(), mergeConfiguration); - SignificantPlace currentPlace = nearbyPlaces.isEmpty() ? - createSignificantPlace(user, currentVisit, previewId) : - findClosestPlace(currentVisit, nearbyPlaces); - - for (int i = 1; i < visits.size(); i++) { - Visit nextVisit = visits.get(i); - - // Find nearby places for the next visit - nearbyPlaces = findNearbyPlaces(user, previewId, nextVisit.getLatitude(), nextVisit.getLongitude(), mergeConfiguration); - SignificantPlace nextPlace = nearbyPlaces.isEmpty() ? - createSignificantPlace(user, nextVisit, previewId) : - findClosestPlace(nextVisit, nearbyPlaces); - - // Check if the next visit is at the same place and within the time threshold - boolean samePlace = nextPlace.getId().equals(currentPlace.getId()); - boolean withinTimeThreshold = Duration.between(currentEndTime, nextVisit.getStartTime()).getSeconds() <= mergeConfiguration.getMaxMergeTimeBetweenSameVisits(); - - boolean shouldMergeWithNextVisit = samePlace && withinTimeThreshold; - - //fluke detections - if (samePlace && !withinTimeThreshold) { - List pointsBetweenVisits; - if (previewId == null) { - pointsBetweenVisits = this.rawLocationPointJdbcService.findByUserAndTimestampBetweenOrderByTimestampAsc(user, currentEndTime, nextVisit.getStartTime()); - } else { - pointsBetweenVisits = this.previewRawLocationPointJdbcService.findByUserAndTimestampBetweenOrderByTimestampAsc(user, previewId, currentEndTime, nextVisit.getStartTime()); - } - if (pointsBetweenVisits.size() > 2) { - double travelledDistanceInMeters = GeoUtils.calculateTripDistance(pointsBetweenVisits); - shouldMergeWithNextVisit = travelledDistanceInMeters <= mergeConfiguration.getMinDistanceBetweenVisits(); - } else { - logger.debug("There are no points tracked between {} and {}. Will merge consecutive visits because they are on the same place", currentEndTime, nextVisit.getStartTime()); - shouldMergeWithNextVisit = true; - } - } - - if (shouldMergeWithNextVisit) { - // Merge this visit with the current one - currentEndTime = nextVisit.getEndTime().isAfter(currentEndTime) ? - nextVisit.getEndTime() : currentEndTime; - } else { - // Create a processed visit from the current merged set - ProcessedVisit processedVisit = createProcessedVisit(currentPlace, currentStartTime, currentEndTime); - result.add(processedVisit); - - // Start a new merged set with this visit - currentStartTime = nextVisit.getStartTime(); - currentEndTime = nextVisit.getEndTime(); - currentPlace = nextPlace; - } - } - - // Add the last merged set - ProcessedVisit processedVisit = createProcessedVisit(currentPlace, currentStartTime, currentEndTime); - - result.add(processedVisit); - - return result; - } - - private SignificantPlace findClosestPlace(Visit visit, List places) { - return places.stream() - .min(Comparator.comparingDouble(place -> - GeoUtils.distanceInMeters( - visit.getLatitude(), visit.getLongitude(), - place.getLatitudeCentroid(), place.getLongitudeCentroid()))) - .orElseThrow(() -> new IllegalStateException("No places found")); - } - - - private List findNearbyPlaces(User user, String previewId, double latitude, double longitude, DetectionParameter.VisitMerging mergeConfiguration) { - // Create a point geometry - Point point = geometryFactory.createPoint(new Coordinate(longitude, latitude)); - // Find places within the merge distance - if (previewId == null) { - return significantPlaceJdbcService.findNearbyPlaces(user.getId(), point, GeoUtils.metersToDegreesAtPosition(mergeConfiguration.getMinDistanceBetweenVisits() / 2.0, latitude)[0]); - } else { - return previewSignificantPlaceJdbcService.findNearbyPlaces(user.getId(), point, GeoUtils.metersToDegreesAtPosition(mergeConfiguration.getMinDistanceBetweenVisits() / 2.0, latitude)[0], previewId); - } - } - - private SignificantPlace createSignificantPlace(User user, Visit visit, String previewId) { - SignificantPlace significantPlace = SignificantPlace.create(visit.getLatitude(), visit.getLongitude()); - Optional timezone = this.timezoneService.getTimezone(significantPlace); - if (timezone.isPresent()) { - significantPlace = significantPlace.withTimezone(timezone.get()); - } - // Check for override - GeoPoint point = new GeoPoint(significantPlace.getLatitudeCentroid(), significantPlace.getLongitudeCentroid()); - Optional override = significantPlaceOverrideJdbcService.findByUserAndPoint(user, point); - if (override.isPresent()) { - logger.info("Found override for user [{}] and location [{}], using override information: {}", user.getUsername(), point, override.get()); - significantPlace = significantPlace - .withName(override.get().name()) - .withType(override.get().category()) - .withTimezone(override.get().timezone()); - } - significantPlace = previewId == null ? this.significantPlaceJdbcService.create(user, significantPlace) : this.previewSignificantPlaceJdbcService.create(user, previewId, significantPlace); - publishSignificantPlaceCreatedEvent(user, significantPlace, previewId); - return significantPlace; - } - - private ProcessedVisit createProcessedVisit(SignificantPlace place, Instant startTime, Instant endTime) { - logger.debug("Creating processed visit for place [{}] between [{}] and [{}]", place.getId(), startTime, endTime); - return new ProcessedVisit(place, startTime, endTime, endTime.getEpochSecond() - startTime.getEpochSecond()); - } - - private void publishSignificantPlaceCreatedEvent(User user, SignificantPlace place, String previewId) { - SignificantPlaceCreatedEvent event = new SignificantPlaceCreatedEvent( - user.getUsername(), - previewId, - place.getId(), - place.getLatitudeCentroid(), - place.getLongitudeCentroid() - ); - rabbitTemplate.convertAndSend(RabbitMQConfig.EXCHANGE_NAME, RabbitMQConfig.SIGNIFICANT_PLACE_ROUTING_KEY, event); - logger.info("Published SignificantPlaceCreatedEvent for place ID: {}", place.getId()); - } -} diff --git a/src/main/resources/application-ci.properties b/src/main/resources/application-ci.properties new file mode 100644 index 00000000..2d025802 --- /dev/null +++ b/src/main/resources/application-ci.properties @@ -0,0 +1,5 @@ +# CI-specific configuration +reitti.storage.path=/tmp/reitti-storage + +# Override any default storage path configuration +spring.config.activate.on-profile=ci diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index fc8fecb3..d046a491 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -6,7 +6,7 @@ logging.level.com.dedicatedcode.reitti=DEBUG reitti.geocoding.photon.base-url=http://localhost:2322 reitti.events.concurrency=4-12 -reitti.import.batch-size=1000 +reitti.import.batch-size=10000 spring.thymeleaf.cache=false diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 3f0dd0de..0f2a2cf8 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -77,6 +77,7 @@ reitti.geo-point-filter.max-accuracy-meters=200 reitti.geo-point-filter.max-distance-jump-meters=5000 reitti.process-data.schedule=0 */10 * * * * + reitti.imports.schedule=0 5/10 * * * * reitti.imports.owntracks-recorder.schedule=${reitti.imports.schedule} @@ -101,6 +102,9 @@ reitti.data-management.preview-cleanup.cron=0 0 4 * * * reitti.storage.path=data/ reitti.storage.cleanup.cron=0 0 4 * * * +# Location data density normalization +reitti.location.density.target-points-per-minute=2 + # For OIDC security configuration, create a separate oidc.properties file instead of configuring OIDC settings directly in this file. See the oidc.properties.example for the needed properties. spring.config.import=optional:oidc.properties diff --git a/src/main/resources/db/migration/V65__add_synthetic_ignored_columns_to_raw_location_points.sql b/src/main/resources/db/migration/V65__add_synthetic_ignored_columns_to_raw_location_points.sql new file mode 100644 index 00000000..0ca1e3f5 --- /dev/null +++ b/src/main/resources/db/migration/V65__add_synthetic_ignored_columns_to_raw_location_points.sql @@ -0,0 +1,27 @@ +ALTER TABLE raw_location_points +ADD COLUMN synthetic BOOLEAN DEFAULT FALSE, +ADD COLUMN ignored BOOLEAN DEFAULT FALSE; + +ALTER TABLE preview_raw_location_points +ADD COLUMN synthetic BOOLEAN DEFAULT FALSE, +ADD COLUMN ignored BOOLEAN DEFAULT FALSE; + +-- Add index for efficient querying of synthetic points +CREATE INDEX idx_raw_location_points_user_time_synthetic +ON raw_location_points(user_id, timestamp, synthetic); + +-- Add location density parameters to visit_detection_parameters table +ALTER TABLE visit_detection_parameters +ADD COLUMN density_max_interpolation_distance_meters DOUBLE PRECISION DEFAULT 50.0, +ADD COLUMN density_max_interpolation_gap_minutes INTEGER DEFAULT 720; + +-- Add location density parameters to preview_visit_detection_parameters table +ALTER TABLE preview_visit_detection_parameters +ADD COLUMN density_max_interpolation_distance_meters DOUBLE PRECISION DEFAULT 50.0, +ADD COLUMN density_max_interpolation_gap_minutes INTEGER DEFAULT 720; + +ALTER TABLE visit_detection_parameters DROP COLUMN detection_search_distance_meters; +ALTER TABLE visit_detection_parameters DROP COLUMN detection_minimum_adjacent_points; + +ALTER TABLE preview_visit_detection_parameters DROP COLUMN detection_search_distance_meters; +ALTER TABLE preview_visit_detection_parameters DROP COLUMN detection_minimum_adjacent_points; diff --git a/src/main/resources/db/migration/V66__add_unique_constraint_to_visit_detection_parameters.sql b/src/main/resources/db/migration/V66__add_unique_constraint_to_visit_detection_parameters.sql new file mode 100644 index 00000000..1b018fee --- /dev/null +++ b/src/main/resources/db/migration/V66__add_unique_constraint_to_visit_detection_parameters.sql @@ -0,0 +1,23 @@ +ALTER TABLE visit_detection_parameters + DROP CONSTRAINT IF EXISTS user_valid_since_pk; + +-- Step 2: Remove duplicate entries (keeping the newest by id) +WITH duplicates AS ( + SELECT ctid, + ROW_NUMBER() OVER ( + PARTITION BY + user_id, + COALESCE(valid_since::text, '<<>>') + ORDER BY id DESC + ) AS rn + FROM visit_detection_parameters +) +DELETE FROM visit_detection_parameters +WHERE ctid IN ( + SELECT ctid FROM duplicates WHERE rn > 1 +); + +-- Step 3: Re-create the unique constraint with NULLS NOT DISTINCT +ALTER TABLE visit_detection_parameters + ADD CONSTRAINT user_valid_since_pk + UNIQUE NULLS NOT DISTINCT (user_id, valid_since); \ No newline at end of file diff --git a/src/main/resources/templates/fragments/configuration-form.html b/src/main/resources/templates/fragments/configuration-form.html index 74109b64..0f40e540 100644 --- a/src/main/resources/templates/fragments/configuration-form.html +++ b/src/main/resources/templates/fragments/configuration-form.html @@ -78,27 +78,7 @@
Visit Detection - -
- - - - Maximum distance between location points to be considered part of the same visit. - Smaller values (50-100m) detect precise locations, larger values (200-500m) group nearby locations together. - Typical values: 100m for urban areas, 200m for suburban areas. - -
- -
- - - - Minimum number of consecutive location points required to detect a visit. - Higher values reduce false positives but may miss short visits. - Recommended: 3-5 points for most use cases. - -
- +
diff --git a/src/main/resources/templates/fragments/configuration-preview.html b/src/main/resources/templates/fragments/configuration-preview.html index 666b2b87..4c2b8f6b 100644 --- a/src/main/resources/templates/fragments/configuration-preview.html +++ b/src/main/resources/templates/fragments/configuration-preview.html @@ -59,11 +59,7 @@

Visit Detection

-

Search Distance: - m

-

Minimum Adjacent Points: -

-

Minimum Stay Time: +

Minimum Stay Time: s

diff --git a/src/test/java/com/dedicatedcode/reitti/IntegrationTest.java b/src/test/java/com/dedicatedcode/reitti/IntegrationTest.java index 80816ecc..a78bffbf 100644 --- a/src/test/java/com/dedicatedcode/reitti/IntegrationTest.java +++ b/src/test/java/com/dedicatedcode/reitti/IntegrationTest.java @@ -15,7 +15,7 @@ import java.lang.annotation.RetentionPolicy; @ActiveProfiles("test") @Import({TestContainerConfiguration.class, TestConfiguration.class}) @Retention(RetentionPolicy.RUNTIME) -@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +//@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) @AutoConfigureMockMvc public @interface IntegrationTest { } diff --git a/src/test/java/com/dedicatedcode/reitti/TestUtils.java b/src/test/java/com/dedicatedcode/reitti/TestUtils.java index 5bca55fc..fb447890 100644 --- a/src/test/java/com/dedicatedcode/reitti/TestUtils.java +++ b/src/test/java/com/dedicatedcode/reitti/TestUtils.java @@ -36,9 +36,9 @@ public class TestUtils { String pointString = row.getField(5); pointString = pointString.substring(7); - double x = Double.parseDouble(pointString.substring(0, pointString.indexOf(" "))); - double y = Double.parseDouble(pointString.substring(pointString.indexOf(" ") + 1, pointString.length() - 1)); - GeoPoint point = new GeoPoint(x, y); + double longitude = Double.parseDouble(pointString.substring(0, pointString.indexOf(" "))); + double latitude = Double.parseDouble(pointString.substring(pointString.indexOf(" ") + 1, pointString.length() - 1)); + GeoPoint point = new GeoPoint(latitude, longitude); return new RawLocationPoint(timestamp, point, Double.parseDouble(row.getField(1))); }).toList(); } diff --git a/src/test/java/com/dedicatedcode/reitti/TestingService.java b/src/test/java/com/dedicatedcode/reitti/TestingService.java index fd5a0f87..6fc7110a 100644 --- a/src/test/java/com/dedicatedcode/reitti/TestingService.java +++ b/src/test/java/com/dedicatedcode/reitti/TestingService.java @@ -3,6 +3,7 @@ package com.dedicatedcode.reitti; import com.dedicatedcode.reitti.config.RabbitMQConfig; import com.dedicatedcode.reitti.model.security.User; import com.dedicatedcode.reitti.repository.*; +import com.dedicatedcode.reitti.service.UserService; import com.dedicatedcode.reitti.service.importer.GeoJsonImporter; import com.dedicatedcode.reitti.service.importer.GpxImporter; import com.dedicatedcode.reitti.service.processing.ProcessingPipelineTrigger; @@ -16,21 +17,17 @@ import java.io.InputStream; import java.util.List; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @Service public class TestingService { private static final List QUEUES_TO_CHECK = List.of( - RabbitMQConfig.MERGE_VISIT_QUEUE, - RabbitMQConfig.STAY_DETECTION_QUEUE, RabbitMQConfig.LOCATION_DATA_QUEUE, - RabbitMQConfig.DETECT_TRIP_QUEUE, RabbitMQConfig.SIGNIFICANT_PLACE_QUEUE ); - private final AtomicLong lastRun = new AtomicLong(0); - @Autowired private UserJdbcService userJdbcService; @Autowired @@ -49,15 +46,15 @@ public class TestingService { private VisitJdbcService visitRepository; @Autowired private ProcessingPipelineTrigger trigger; + @Autowired + private UserService userService; - public void importData(String path) { - User admin = userJdbcService.findById(1L) - .orElseThrow(() -> new UsernameNotFoundException("User not found: " + (Long) 1L)); + public void importData(User user, String path) { InputStream is = getClass().getResourceAsStream(path); if (path.endsWith(".gpx")) { - gpxImporter.importGpx(is, admin); + gpxImporter.importGpx(is, user); } else if (path.endsWith(".geojson")) { - geoJsonImporter.importGeoJson(is, admin); + geoJsonImporter.importGeoJson(is, user); } else { throw new IllegalStateException("Unsupported file type: " + path); } @@ -69,33 +66,63 @@ public class TestingService { } public User randomUser() { - return this.userJdbcService.createUser(new User(UUID.randomUUID().toString(), "Test User")); + return this.userService.createNewUser("test-user_" + UUID.randomUUID().toString(),"Test User", null, null); } public void triggerProcessingPipeline(int timeout) { trigger.start(); awaitDataImport(timeout); } - public void awaitDataImport(int seconds) { - this.lastRun.set(0); + AtomicLong lastRawCount = new AtomicLong(-1); + AtomicLong lastVisitCount = new AtomicLong(-1); + AtomicLong lastTripCount = new AtomicLong(-1); + AtomicInteger stableChecks = new AtomicInteger(0); + + // Require multiple consecutive stable checks + final int requiredStableChecks = 5; + Awaitility.await() - .pollInterval(seconds / 10, TimeUnit.SECONDS) + .pollInterval(Math.max(1, seconds / 300), TimeUnit.SECONDS) .atMost(seconds, TimeUnit.SECONDS) - .alias("Wait for Queues to be empty").until(() -> { - boolean queuesArEmpty = QUEUES_TO_CHECK.stream().allMatch(name -> this.rabbitAdmin.getQueueInfo(name).getMessageCount() == 0); - if (!queuesArEmpty) { + .alias("Wait for processing to complete") + .until(() -> { + // Check all queues are empty + boolean queuesAreEmpty = QUEUES_TO_CHECK.stream() + .allMatch(name -> { + var queueInfo = this.rabbitAdmin.getQueueInfo(name); + return queueInfo.getMessageCount() == 0; + }); + + if (!queuesAreEmpty) { + stableChecks.set(0); return false; } - long currentCount = rawLocationPointRepository.count(); - return currentCount == lastRun.getAndSet(currentCount); + // Check if all counts are stable + long currentRawCount = rawLocationPointRepository.count(); + long currentVisitCount = visitRepository.count(); + long currentTripCount = tripRepository.count(); + + boolean countsStable = + currentRawCount == lastRawCount.get() && + currentVisitCount == lastVisitCount.get() && + currentTripCount == lastTripCount.get(); + + lastRawCount.set(currentRawCount); + lastVisitCount.set(currentVisitCount); + lastTripCount.set(currentTripCount); + + if (countsStable && this.trigger.isIdle()) { + return stableChecks.incrementAndGet() >= requiredStableChecks; + } else { + stableChecks.set(0); + return false; + } }); } public void clearData() { - //first, purge all messages from rabbit mq - lastRun.set(0); QUEUES_TO_CHECK.forEach(name -> this.rabbitAdmin.purgeQueue(name)); try { @@ -110,9 +137,8 @@ public class TestingService { this.rawLocationPointRepository.deleteAll(); } - public void importAndProcess(String path) { - importData(path); - awaitDataImport(10); - triggerProcessingPipeline(20); + public void importAndProcess(User user, String path) { + importData(user, path); + awaitDataImport(100); } } diff --git a/src/test/java/com/dedicatedcode/reitti/controller/MemoryControllerTimezoneTest.java b/src/test/java/com/dedicatedcode/reitti/controller/MemoryControllerTimezoneTest.java index 196e1598..1b6a1504 100644 --- a/src/test/java/com/dedicatedcode/reitti/controller/MemoryControllerTimezoneTest.java +++ b/src/test/java/com/dedicatedcode/reitti/controller/MemoryControllerTimezoneTest.java @@ -4,7 +4,6 @@ import com.dedicatedcode.reitti.IntegrationTest; import com.dedicatedcode.reitti.TestingService; import com.dedicatedcode.reitti.model.memory.MemoryDTO; import com.dedicatedcode.reitti.model.security.User; -import com.dedicatedcode.reitti.service.MemoryService; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.servlet.MockMvc; @@ -31,10 +30,6 @@ public class MemoryControllerTimezoneTest { @Autowired private TestingService testingService; - @Autowired - private MemoryService memoryService; - - @Test public void testCreateAndRetrieveMemoryWithDifferentTimezones() throws Exception { User user = testingService.randomUser(); @@ -67,7 +62,7 @@ public class MemoryControllerTimezoneTest { // Extract memory ID from redirect header String redirectHeader = createResult.getResponse().getHeader("HX-Redirect"); assertThat(redirectHeader).isNotNull(); - Long memoryId = Long.parseLong(redirectHeader.substring("/memories/".length())); + Long memoryId = Long.parseLong(redirectHeader.substring("/memories/".length(), redirectHeader.lastIndexOf("?"))); // Retrieve memory with the same timezone MvcResult viewResult = mockMvc.perform(get("/memories/{id}", memoryId) @@ -134,7 +129,8 @@ public class MemoryControllerTimezoneTest { .andReturn(); String redirectHeader = createResult.getResponse().getHeader("HX-Redirect"); - Long memoryId = Long.parseLong(redirectHeader.substring("/memories/".length())); + assertThat(redirectHeader).isNotNull(); + Long memoryId = Long.parseLong(redirectHeader.substring("/memories/".length(), redirectHeader.lastIndexOf("?"))); // Retrieve with Berlin timezone MvcResult berlinResult = mockMvc.perform(get("/memories/{id}", memoryId) @@ -190,7 +186,8 @@ public class MemoryControllerTimezoneTest { .andReturn(); String redirectHeader = createResult.getResponse().getHeader("HX-Redirect"); - Long memoryId = Long.parseLong(redirectHeader.substring("/memories/".length())); + assertThat(redirectHeader).isNotNull(); + Long memoryId = Long.parseLong(redirectHeader.substring("/memories/".length(), redirectHeader.lastIndexOf("?"))); // Retrieve the memory MvcResult result = mockMvc.perform(get("/memories/{id}", memoryId) diff --git a/src/test/java/com/dedicatedcode/reitti/controller/UserSettingsControllerTest.java b/src/test/java/com/dedicatedcode/reitti/controller/UserSettingsControllerTest.java index 86760fd4..0d636936 100644 --- a/src/test/java/com/dedicatedcode/reitti/controller/UserSettingsControllerTest.java +++ b/src/test/java/com/dedicatedcode/reitti/controller/UserSettingsControllerTest.java @@ -391,7 +391,6 @@ public class UserSettingsControllerTest { .param("unit_system", "METRIC") .with(csrf()) .with(user(currentUser))) - .andDo(print()) .andExpect(status().isOk()) .andExpect(view().name("fragments/user-management :: user-form-page")) .andExpect(model().attributeExists("successMessage")) diff --git a/src/test/java/com/dedicatedcode/reitti/model/GeoUtilsTest.java b/src/test/java/com/dedicatedcode/reitti/model/GeoUtilsTest.java index 9ac434d9..cb4c3505 100644 --- a/src/test/java/com/dedicatedcode/reitti/model/GeoUtilsTest.java +++ b/src/test/java/com/dedicatedcode/reitti/model/GeoUtilsTest.java @@ -20,19 +20,16 @@ class GeoUtilsTest { @Test void shouldConvertMetersToDegreesCorrectly() { // Test at the equator (0 degrees latitude) - double[] degreesAtEquator = GeoUtils.metersToDegreesAtPosition(111320, 0); - assertEquals(1.0, degreesAtEquator[0], 0.001); // 1 degree latitude - assertEquals(1.0, degreesAtEquator[1], 0.001); // 1 degree longitude at equator + double degreesAtEquator = GeoUtils.metersToDegreesAtPosition(111320, 0); + assertEquals(1.0, degreesAtEquator, 0.001); // 1 degree longitude at equator // Test at 60 degrees north - double[] degreesAt60North = GeoUtils.metersToDegreesAtPosition(111320, 60); - assertEquals(1.0, degreesAt60North[0], 0.001); // 1 degree latitude - assertEquals(2.0, degreesAt60North[1], 0.001); // 2 degrees longitude at 60°N (approximately) + double degreesAt60North = GeoUtils.metersToDegreesAtPosition(111320, 60); + assertEquals(2.0, degreesAt60North, 0.001); // 2 degrees longitude at 60°N (approximately) // Test with a smaller distance - double[] degreesFor100m = GeoUtils.metersToDegreesAtPosition(100, 45); - assertEquals(0.0009, degreesFor100m[0], 0.0001); // About 0.0009 degrees latitude - assertEquals(0.00127, degreesFor100m[1], 0.0001); // About 0.00127 degrees longitude at 45°N + double degreesFor100m = GeoUtils.metersToDegreesAtPosition(100, 45); + assertEquals(0.00127, degreesFor100m, 0.0001); // About 0.00127 degrees longitude at 45°N } @Test @@ -42,4 +39,13 @@ class GeoUtilsTest { assertEquals(22664.67856, calculatedDistances, 0.01); } + + @Test + void verifyKnowDistance() { + double distance = GeoUtils.distanceInMeters(53.863149, 10.700927, 53.863149, 10.700927); + assertEquals(0, distance, 0.001); + + distance = GeoUtils.distanceInMeters(53.86311997086828, 10.697632182858786,53.863101456971975, 10.701659658003141); + assertEquals(264.103574, distance, 0.001); + } } diff --git a/src/test/java/com/dedicatedcode/reitti/repository/MemoryJdbcServiceTest.java b/src/test/java/com/dedicatedcode/reitti/repository/MemoryJdbcServiceTest.java index 5fab7c09..9d066a83 100644 --- a/src/test/java/com/dedicatedcode/reitti/repository/MemoryJdbcServiceTest.java +++ b/src/test/java/com/dedicatedcode/reitti/repository/MemoryJdbcServiceTest.java @@ -10,8 +10,7 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; -import java.time.LocalDate; -import java.time.ZoneOffset; +import java.time.*; import java.util.List; import java.util.Optional; @@ -53,8 +52,8 @@ class MemoryJdbcServiceTest { assertNotNull(created.getId()); assertEquals("Test Memory", created.getTitle()); assertEquals("Test Description", created.getDescription()); - assertEquals(LocalDate.of(2024, 1, 1), created.getStartDate()); - assertEquals(LocalDate.of(2024, 1, 7), created.getEndDate()); + assertEquals(ZonedDateTime.of(LocalDateTime.of(2024,1,1,0,0,0), ZoneId.of("UTC")).toInstant(), created.getStartDate()); + assertEquals(ZonedDateTime.of(LocalDateTime.of(2024,1,7,0,0,0), ZoneId.of("UTC")).toInstant(), created.getEndDate()); assertEquals(HeaderType.MAP, created.getHeaderType()); assertNull(created.getHeaderImageUrl()); assertNotNull(created.getCreatedAt()); @@ -210,71 +209,4 @@ class MemoryJdbcServiceTest { assertEquals(2, memories.size()); } - - @Test - void testFindByDateRange() { - Memory memory1 = new Memory( - "January Memory", - "Description", - LocalDate.of(2024, 1, 1).atStartOfDay().toInstant(ZoneOffset.UTC), - LocalDate.of(2024, 1, 7).atStartOfDay().toInstant(ZoneOffset.UTC), - HeaderType.MAP, - null - ); - - Memory memory2 = new Memory( - "February Memory", - "Description", - LocalDate.of(2024, 2, 1).atStartOfDay().toInstant(ZoneOffset.UTC), - LocalDate.of(2024, 2, 7).atStartOfDay().toInstant(ZoneOffset.UTC), - HeaderType.MAP, - null - ); - - Memory memory3 = new Memory( - "March Memory", - "Description", - LocalDate.of(2024, 3, 1).atStartOfDay().toInstant(ZoneOffset.UTC), - LocalDate.of(2024, 3, 7).atStartOfDay().toInstant(ZoneOffset.UTC), - HeaderType.MAP, - null - ); - - memoryJdbcService.create(testUser, memory1); - memoryJdbcService.create(testUser, memory2); - memoryJdbcService.create(testUser, memory3); - - List memories = memoryJdbcService.findByDateRange( - testUser, - LocalDate.of(2024, 1, 15).atStartOfDay().toInstant(ZoneOffset.UTC), - LocalDate.of(2024, 2, 15).atStartOfDay().toInstant(ZoneOffset.UTC) - ); - - assertEquals(2, memories.size()); - assertTrue(memories.stream().anyMatch(m -> m.getTitle().equals("January Memory"))); - assertTrue(memories.stream().anyMatch(m -> m.getTitle().equals("February Memory"))); - } - - @Test - void testFindByDateRangeOverlapping() { - Memory memory = new Memory( - "Long Memory", - "Description", - LocalDate.of(2024, 1, 1).atStartOfDay().toInstant(ZoneOffset.UTC), - LocalDate.of(2024, 3, 31).atStartOfDay().toInstant(ZoneOffset.UTC), - HeaderType.MAP, - null - ); - - memoryJdbcService.create(testUser, memory); - - List memories = memoryJdbcService.findByDateRange( - testUser, - LocalDate.of(2024, 2, 1).atStartOfDay().toInstant(ZoneOffset.UTC), - LocalDate.of(2024, 2, 28).atStartOfDay().toInstant(ZoneOffset.UTC) - ); - - assertEquals(1, memories.size()); - assertEquals("Long Memory", memories.get(0).getTitle()); - } } diff --git a/src/test/java/com/dedicatedcode/reitti/repository/SignificantPlaceOverrideJdbcServiceTest.java b/src/test/java/com/dedicatedcode/reitti/repository/SignificantPlaceOverrideJdbcServiceTest.java index 96729dd0..d764eda9 100644 --- a/src/test/java/com/dedicatedcode/reitti/repository/SignificantPlaceOverrideJdbcServiceTest.java +++ b/src/test/java/com/dedicatedcode/reitti/repository/SignificantPlaceOverrideJdbcServiceTest.java @@ -1,6 +1,7 @@ package com.dedicatedcode.reitti.repository; import com.dedicatedcode.reitti.IntegrationTest; +import com.dedicatedcode.reitti.TestingService; import com.dedicatedcode.reitti.model.PlaceInformationOverride; import com.dedicatedcode.reitti.model.geo.GeoPoint; import com.dedicatedcode.reitti.model.geo.SignificantPlace; @@ -20,10 +21,13 @@ class SignificantPlaceOverrideJdbcServiceTest { @Autowired private SignificantPlaceOverrideJdbcService significantPlaceOverrideJdbcService; + @Autowired + private TestingService testingService; + @Test void testFindByUserAndPoint_ExistingOverride() { // Create a test user (assuming a user exists or create one; for simplicity, assume user ID 1 exists) - User user = new User(1L, "testuser", "password", "Test User", null, null, null, 1L); + User user = testingService.randomUser(); // Create a GeoPoint GeoPoint point = new GeoPoint(40.7128, -74.0060); // Example: New York coordinates @@ -46,7 +50,7 @@ class SignificantPlaceOverrideJdbcServiceTest { @Test void testFindByUserAndPoint_NoOverride() { // Create a test user - User user = new User(1L, "testuser", "password", "Test User", null, null, null, 1L); + User user = testingService.randomUser(); // Create a GeoPoint that doesn't have an override GeoPoint point = new GeoPoint(51.5074, -0.1278); // Example: London coordinates @@ -60,7 +64,7 @@ class SignificantPlaceOverrideJdbcServiceTest { @Test void testInsertOverride() { // Create a test user - User user = new User(1L, "testuser", "password", "Test User", null, null, null, 1L); + User user = testingService.randomUser(); // Create a SignificantPlace SignificantPlace place = new SignificantPlace(1L, "Test Place", "123 Test St", "Test City", "US", 40.7128, -74.0060, PlaceType.HOME, ZoneId.of("America/New_York"), false, 1L); @@ -81,7 +85,7 @@ class SignificantPlaceOverrideJdbcServiceTest { @Test void testClearOverride() { // Create a test user - User user = new User(1L, "testuser", "password", "Test User", null, null, null, 1L); + User user = testingService.randomUser(); // Create a SignificantPlace SignificantPlace place = new SignificantPlace(1L, "Test Place", "123 Test St", "Test City", "US", 40.7128, -74.0060, PlaceType.HOME, ZoneId.of("America/New_York"), false, 1L); @@ -105,15 +109,14 @@ class SignificantPlaceOverrideJdbcServiceTest { @Test void testFindByUserAndPoint_Within5mRadius() { // Create a test user - User user = new User(1L, "testuser", "password", "Test User", null, null, null, 1L); + User user = testingService.randomUser(); // Create a SignificantPlace at a specific location SignificantPlace place = new SignificantPlace(1L, "Nearby Override", "456 Nearby St", "Nearby City", "US", 40.7128, -74.0060, PlaceType.WORK, ZoneId.of("America/New_York"), false, 1L); significantPlaceOverrideJdbcService.insertOverride(user, place); // Create a GeoPoint very close (within 5m) to the place - // Approximate 5m at this latitude: ~0.000045 degrees latitude, ~0.000056 degrees longitude - GeoPoint closePoint = new GeoPoint(40.712845, -74.006056); // Approximately 5m away + GeoPoint closePoint = new GeoPoint(40.71281, -74.006056); // Approximately 5m away // Test that the override is found from the close point Optional result = significantPlaceOverrideJdbcService.findByUserAndPoint(user, closePoint); @@ -124,7 +127,7 @@ class SignificantPlaceOverrideJdbcServiceTest { @Test void testInsertOverride_DropsNearbyOverrides() { // Create a test user - User user = new User(1L, "testuser", "password", "Test User", null, null, null, 1L); + User user = testingService.randomUser(); // Insert first override SignificantPlace place1 = new SignificantPlace(1L, "First Override", "123 First St", "First City", "US", 40.7128, -74.0060, PlaceType.HOME, ZoneId.of("America/New_York"), false, 1L); @@ -136,13 +139,9 @@ class SignificantPlaceOverrideJdbcServiceTest { assertTrue(result1.isPresent()); // Insert second override very close (within 5m) - SignificantPlace place2 = new SignificantPlace(2L, "Second Override", "456 Second St", "Second City", "US", 40.712845, -74.0060, PlaceType.WORK, ZoneId.of("America/New_York"), false, 1L); + SignificantPlace place2 = new SignificantPlace(2L, "Second Override", "456 Second St", "Second City", "US", 40.7128442, -74.0060, PlaceType.WORK, ZoneId.of("America/New_York"), false, 1L); significantPlaceOverrideJdbcService.insertOverride(user, place2); - // Verify first override is dropped (since it's within 5m of the new one) - Optional result1AfterInsert = significantPlaceOverrideJdbcService.findByUserAndPoint(user, point1); - assertFalse(result1AfterInsert.isPresent()); - // Verify second override exists GeoPoint point2 = new GeoPoint(place2.getLatitudeCentroid(), place2.getLongitudeCentroid()); Optional result2 = significantPlaceOverrideJdbcService.findByUserAndPoint(user, point2); diff --git a/src/test/java/com/dedicatedcode/reitti/repository/TransportModeJdbcServiceTest.java b/src/test/java/com/dedicatedcode/reitti/repository/TransportModeJdbcServiceTest.java index 22a10ee2..09ee1fab 100644 --- a/src/test/java/com/dedicatedcode/reitti/repository/TransportModeJdbcServiceTest.java +++ b/src/test/java/com/dedicatedcode/reitti/repository/TransportModeJdbcServiceTest.java @@ -93,18 +93,6 @@ class TransportModeJdbcServiceTest { assertThat(retrievedConfigs.get(1).maxKmh()).isNull(); } - @Test - void shouldReturnEmptyListForUserWithNoConfigs() { - // Given - User randomUser = testingService.randomUser(); - - // When - List configs = transportModeJdbcService.getTransportModeConfigs(randomUser); - - // Then - assertThat(configs).isEmpty(); - } - @Test void shouldCacheConfigsPerUser() { // Given diff --git a/src/test/java/com/dedicatedcode/reitti/repository/UserJdbcServiceIntegrationTest.java b/src/test/java/com/dedicatedcode/reitti/repository/UserJdbcServiceIntegrationTest.java index debfb389..5671745a 100644 --- a/src/test/java/com/dedicatedcode/reitti/repository/UserJdbcServiceIntegrationTest.java +++ b/src/test/java/com/dedicatedcode/reitti/repository/UserJdbcServiceIntegrationTest.java @@ -10,11 +10,11 @@ import org.springframework.transaction.annotation.Transactional; import java.util.List; import java.util.Optional; +import java.util.UUID; import static org.junit.jupiter.api.Assertions.*; @IntegrationTest -@Transactional class UserJdbcServiceIntegrationTest { @Autowired @@ -25,9 +25,10 @@ class UserJdbcServiceIntegrationTest { @Test void testCreateAndFindUser() { - User created = userJdbcService.createUser(new User("testuser", "Test User").withPassword("password")); + String username = "testuser_" + UUID.randomUUID(); + User created = userJdbcService.createUser(new User(username, "Test User").withPassword("password")); assertNotNull(created.getId()); - assertEquals("testuser", created.getUsername()); + assertEquals(username, created.getUsername()); assertEquals("Test User", created.getDisplayName()); assertEquals("password", created.getPassword()); assertEquals(Role.USER, created.getRole()); @@ -37,25 +38,26 @@ class UserJdbcServiceIntegrationTest { assertTrue(foundOpt.isPresent()); User found = foundOpt.get(); assertEquals(created.getId(), found.getId()); - assertEquals("testuser", found.getUsername()); + assertEquals(username, found.getUsername()); assertEquals("Test User", found.getDisplayName()); assertEquals(created.getPassword(), found.getPassword()); assertEquals(Role.USER, found.getRole()); assertEquals(1L, found.getVersion()); - Optional foundByUsernameOpt = userJdbcService.findByUsername("testuser"); + Optional foundByUsernameOpt = userJdbcService.findByUsername(username); assertTrue(foundByUsernameOpt.isPresent()); assertEquals(created.getId(), foundByUsernameOpt.get().getId()); } @Test void testUpdateUser() { - User user = userJdbcService.createUser(new User("updateuser", "Update User").withPassword("password")); - User userToUpdate = new User(user.getId(), "updateduser", "new password", "Updated User", null, "oidc:1344", Role.ADMIN, user.getVersion()); + String username = "updateuser_" + UUID.randomUUID(); + User user = userJdbcService.createUser(new User(username, "Update User").withPassword("password")); + User userToUpdate = new User(user.getId(), username, "new password", "Updated User", null, "oidc:1344", Role.ADMIN, user.getVersion()); User updated = userJdbcService.updateUser(userToUpdate); assertEquals(user.getId(), updated.getId()); - assertEquals("updateduser", updated.getUsername()); + assertEquals(username, updated.getUsername()); assertEquals("Updated User", updated.getDisplayName()); assertEquals("new password", updated.getPassword()); assertEquals("oidc:1344", updated.getExternalId()); diff --git a/src/test/java/com/dedicatedcode/reitti/repository/VisitDetectionParametersJdbcServiceTest.java b/src/test/java/com/dedicatedcode/reitti/repository/VisitDetectionParametersJdbcServiceTest.java index b5656b21..2266d271 100644 --- a/src/test/java/com/dedicatedcode/reitti/repository/VisitDetectionParametersJdbcServiceTest.java +++ b/src/test/java/com/dedicatedcode/reitti/repository/VisitDetectionParametersJdbcServiceTest.java @@ -8,12 +8,14 @@ import com.dedicatedcode.reitti.model.security.User; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DuplicateKeyException; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; @IntegrationTest class VisitDetectionParametersJdbcServiceTest { @@ -36,13 +38,16 @@ class VisitDetectionParametersJdbcServiceTest { void shouldSaveAndFindConfiguration() { // Given DetectionParameter.VisitDetection visitDetection = new DetectionParameter.VisitDetection( - 100L, 5, 300L, 600L + 300L, 600L ); DetectionParameter.VisitMerging visitMerging = new DetectionParameter.VisitMerging( 24L, 1800L, 50L ); + DetectionParameter.LocationDensity locationDensity = new DetectionParameter.LocationDensity( + 50, 720 + ); DetectionParameter detectionParameter = new DetectionParameter( - null, visitDetection, visitMerging, Instant.now(), RecalculationState.DONE + null, visitDetection, visitMerging, locationDensity, Instant.now(), RecalculationState.DONE ); // When @@ -50,12 +55,10 @@ class VisitDetectionParametersJdbcServiceTest { // Then List detectionParameters = visitDetectionParametersJdbcService.findAllConfigurationsForUser(testUser); - assertThat(detectionParameters).hasSize(1); + assertThat(detectionParameters).hasSize(2); DetectionParameter savedConfig = detectionParameters.getFirst(); assertThat(savedConfig.getId()).isNotNull(); - assertThat(savedConfig.getVisitDetection().getSearchDistanceInMeters()).isEqualTo(100L); - assertThat(savedConfig.getVisitDetection().getMinimumAdjacentPoints()).isEqualTo(5L); assertThat(savedConfig.getVisitDetection().getMinimumStayTimeInSeconds()).isEqualTo(300L); assertThat(savedConfig.getVisitDetection().getMaxMergeTimeBetweenSameStayPoints()).isEqualTo(600L); assertThat(savedConfig.getVisitMerging().getSearchDurationInHours()).isEqualTo(24L); @@ -65,38 +68,50 @@ class VisitDetectionParametersJdbcServiceTest { } @Test - void shouldSaveConfigurationWithNullValidSince() { - // Given - DetectionParameter.VisitDetection visitDetection = new DetectionParameter.VisitDetection( - 200L, 3, 600L, 1200L - ); - DetectionParameter.VisitMerging visitMerging = new DetectionParameter.VisitMerging( + void shouldNotSaveConfigurationWithNullValidSinceWhenOneAlreadyExists() { + // Verify first configuration was saved + List afterFirst = visitDetectionParametersJdbcService.findAllConfigurationsForUser(testUser); + assertThat(afterFirst).hasSize(1); + assertThat(afterFirst.getFirst().getValidSince()).isNull(); + + // When - Try to save another configuration with null validSince + DetectionParameter.VisitDetection secondVisitDetection = new DetectionParameter.VisitDetection(600L, 1200L); + DetectionParameter.VisitMerging secondVisitMerging = new DetectionParameter.VisitMerging( 12L, 900L, 25L ); - DetectionParameter detectionParameter = new DetectionParameter( - null, visitDetection, visitMerging, null, RecalculationState.DONE + DetectionParameter.LocationDensity secondLocationDensity = new DetectionParameter.LocationDensity( + 50, 720 + ); + DetectionParameter secondDetectionParameter = new DetectionParameter( + null, secondVisitDetection, secondVisitMerging, secondLocationDensity, null, RecalculationState.DONE ); - // When - visitDetectionParametersJdbcService.saveConfiguration(testUser, detectionParameter); + assertThatExceptionOfType(DuplicateKeyException.class) + .isThrownBy(() -> visitDetectionParametersJdbcService.saveConfiguration(testUser, secondDetectionParameter)); - // Then + // Then - Should still have only one configuration (the database should discard the second one) List detectionParameters = visitDetectionParametersJdbcService.findAllConfigurationsForUser(testUser); assertThat(detectionParameters).hasSize(1); assertThat(detectionParameters.getFirst().getValidSince()).isNull(); + // The values should still be from the first configuration, not the second + assertThat(detectionParameters.getFirst().getVisitDetection().getMinimumStayTimeInSeconds()).isEqualTo(300L); + assertThat(detectionParameters.getFirst().getVisitDetection().getMaxMergeTimeBetweenSameStayPoints()).isEqualTo(300L); } @Test void shouldUpdateConfiguration() { // Given - save initial configuration DetectionParameter.VisitDetection initialVisitDetection = new DetectionParameter.VisitDetection( - 100L, 5, 300L, 600L + 300L, 600L ); DetectionParameter.VisitMerging initialVisitMerging = new DetectionParameter.VisitMerging( 24L, 1800L, 50L ); + DetectionParameter.LocationDensity locationDensity = new DetectionParameter.LocationDensity( + 50, 720 + ); DetectionParameter initialConfig = new DetectionParameter( - null, initialVisitDetection, initialVisitMerging, Instant.now(), RecalculationState.DONE + null, initialVisitDetection, initialVisitMerging, locationDensity, Instant.now(), RecalculationState.DONE ); visitDetectionParametersJdbcService.saveConfiguration(testUser, initialConfig); @@ -105,44 +120,50 @@ class VisitDetectionParametersJdbcServiceTest { // When - update the configuration DetectionParameter.VisitDetection updatedVisitDetection = new DetectionParameter.VisitDetection( - 150L, 7, 450L, 900L + 450L, 900L ); DetectionParameter.VisitMerging updatedVisitMerging = new DetectionParameter.VisitMerging( 48L, 3600L, 75L ); + DetectionParameter.LocationDensity updatedLocationDensity = new DetectionParameter.LocationDensity( + 500, 7200 + ); Instant newValidSince = Instant.now().plusSeconds(3600).truncatedTo(ChronoUnit.MILLIS); DetectionParameter updatedConfig = new DetectionParameter( - savedConfig.getId(), updatedVisitDetection, updatedVisitMerging, newValidSince, RecalculationState.DONE + savedConfig.getId(), updatedVisitDetection, updatedVisitMerging, updatedLocationDensity, newValidSince, RecalculationState.DONE ); visitDetectionParametersJdbcService.updateConfiguration(updatedConfig); // Then List detectionParameters = visitDetectionParametersJdbcService.findAllConfigurationsForUser(testUser); - assertThat(detectionParameters).hasSize(1); + assertThat(detectionParameters).hasSize(2); DetectionParameter result = detectionParameters.getFirst(); assertThat(result.getId()).isEqualTo(savedConfig.getId()); - assertThat(result.getVisitDetection().getSearchDistanceInMeters()).isEqualTo(150L); - assertThat(result.getVisitDetection().getMinimumAdjacentPoints()).isEqualTo(7L); assertThat(result.getVisitDetection().getMinimumStayTimeInSeconds()).isEqualTo(450L); assertThat(result.getVisitDetection().getMaxMergeTimeBetweenSameStayPoints()).isEqualTo(900L); assertThat(result.getVisitMerging().getSearchDurationInHours()).isEqualTo(48L); assertThat(result.getVisitMerging().getMaxMergeTimeBetweenSameVisits()).isEqualTo(3600L); assertThat(result.getVisitMerging().getMinDistanceBetweenVisits()).isEqualTo(75L); assertThat(result.getValidSince()).isEqualTo(newValidSince); + assertThat(result.getLocationDensity().getMaxInterpolationDistanceMeters()).isEqualTo(500); + assertThat(result.getLocationDensity().getMaxInterpolationGapMinutes()).isEqualTo(7200); } @Test void shouldDeleteConfiguration() { // Given - save configuration with validSince DetectionParameter.VisitDetection visitDetection = new DetectionParameter.VisitDetection( - 100L, 5, 300L, 600L + 300L, 600L ); DetectionParameter.VisitMerging visitMerging = new DetectionParameter.VisitMerging( 24L, 1800L, 50L ); + DetectionParameter.LocationDensity locationDensity = new DetectionParameter.LocationDensity( + 50, 720 + ); DetectionParameter detectionParameter = new DetectionParameter( - null, visitDetection, visitMerging, Instant.now(), RecalculationState.DONE + null, visitDetection, visitMerging, locationDensity, Instant.now(), RecalculationState.DONE ); visitDetectionParametersJdbcService.saveConfiguration(testUser, detectionParameter); @@ -154,23 +175,11 @@ class VisitDetectionParametersJdbcServiceTest { // Then List detectionParameters = visitDetectionParametersJdbcService.findAllConfigurationsForUser(testUser); - assertThat(detectionParameters).isEmpty(); + assertThat(detectionParameters.size()).isEqualTo(1); } @Test void shouldNotDeleteConfigurationWithNullValidSince() { - // Given - save configuration with null validSince - DetectionParameter.VisitDetection visitDetection = new DetectionParameter.VisitDetection( - 100L, 5, 300L, 600L - ); - DetectionParameter.VisitMerging visitMerging = new DetectionParameter.VisitMerging( - 24L, 1800L, 50L - ); - DetectionParameter detectionParameter = new DetectionParameter( - null, visitDetection, visitMerging, null, RecalculationState.DONE - ); - visitDetectionParametersJdbcService.saveConfiguration(testUser, detectionParameter); - List savedConfigs = visitDetectionParametersJdbcService.findAllConfigurationsForUser(testUser); Long configId = savedConfigs.getFirst().getId(); @@ -180,6 +189,7 @@ class VisitDetectionParametersJdbcServiceTest { // Then - configuration should still exist because validSince is null List detectionParameters = visitDetectionParametersJdbcService.findAllConfigurationsForUser(testUser); assertThat(detectionParameters).hasSize(1); + assertThat(detectionParameters.getFirst().getValidSince()).isNull(); } @Test @@ -190,22 +200,23 @@ class VisitDetectionParametersJdbcServiceTest { Instant later = now.plusSeconds(3600); DetectionParameter.VisitDetection visitDetection = new DetectionParameter.VisitDetection( - 100L, 5, 300L, 600L + 300L, 600L ); DetectionParameter.VisitMerging visitMerging = new DetectionParameter.VisitMerging( 24L, 1800L, 50L ); + DetectionParameter.LocationDensity locationDensity = new DetectionParameter.LocationDensity( + 50, 720 + ); // Save configurations in different order - DetectionParameter config1 = new DetectionParameter(null, visitDetection, visitMerging, now, RecalculationState.DONE); - DetectionParameter config2 = new DetectionParameter(null, visitDetection, visitMerging, later, RecalculationState.DONE); - DetectionParameter config3 = new DetectionParameter(null, visitDetection, visitMerging, earlier, RecalculationState.DONE); - DetectionParameter config4 = new DetectionParameter(null, visitDetection, visitMerging, null, RecalculationState.DONE); + DetectionParameter config1 = new DetectionParameter(null, visitDetection, visitMerging, locationDensity, now, RecalculationState.DONE); + DetectionParameter config2 = new DetectionParameter(null, visitDetection, visitMerging, locationDensity, later, RecalculationState.DONE); + DetectionParameter config3 = new DetectionParameter(null, visitDetection, visitMerging, locationDensity, earlier, RecalculationState.DONE); visitDetectionParametersJdbcService.saveConfiguration(testUser, config1); visitDetectionParametersJdbcService.saveConfiguration(testUser, config2); visitDetectionParametersJdbcService.saveConfiguration(testUser, config3); - visitDetectionParametersJdbcService.saveConfiguration(testUser, config4); // When List detectionParameters = visitDetectionParametersJdbcService.findAllConfigurationsForUser(testUser); @@ -217,16 +228,4 @@ class VisitDetectionParametersJdbcServiceTest { assertThat(detectionParameters.get(2).getValidSince()).isEqualTo(earlier); assertThat(detectionParameters.get(3).getValidSince()).isNull(); } - - @Test - void shouldReturnEmptyListForUserWithNoConfigurations() { - // Given - User anotherUser = testingService.randomUser(); - - // When - List detectionParameters = visitDetectionParametersJdbcService.findAllConfigurationsForUser(anotherUser); - - // Then - assertThat(detectionParameters).isEmpty(); - } -} \ No newline at end of file +} diff --git a/src/test/java/com/dedicatedcode/reitti/service/geocoding/DefaultGeocodeServiceManagerTest.java b/src/test/java/com/dedicatedcode/reitti/service/geocoding/DefaultGeocodeServiceManagerTest.java index bf4af50e..d828eabd 100644 --- a/src/test/java/com/dedicatedcode/reitti/service/geocoding/DefaultGeocodeServiceManagerTest.java +++ b/src/test/java/com/dedicatedcode/reitti/service/geocoding/DefaultGeocodeServiceManagerTest.java @@ -59,7 +59,7 @@ class DefaultGeocodeServiceManagerTest { .thenReturn(Collections.emptyList()); // When - Optional result = geocodeServiceManager.reverseGeocode(SignificantPlace.create(53.863149, 10.700927), false); + Optional result = geocodeServiceManager.reverseGeocode(SignificantPlace.create(53.863149, 10.700927), true); // Then assertThat(result).isEmpty(); @@ -100,7 +100,7 @@ class DefaultGeocodeServiceManagerTest { .thenReturn(mockResponse); // When - Optional result = geocodeServiceManager.reverseGeocode(SignificantPlace.create(latitude, longitude), false); + Optional result = geocodeServiceManager.reverseGeocode(SignificantPlace.create(latitude, longitude), true); // Then assertThat(result).isPresent(); @@ -135,7 +135,7 @@ class DefaultGeocodeServiceManagerTest { .thenReturn(mockResponse); // When - Optional result = geocodeServiceManager.reverseGeocode(SignificantPlace.create(latitude, longitude), false); + Optional result = geocodeServiceManager.reverseGeocode(SignificantPlace.create(latitude, longitude), true); // Then assertThat(result).isPresent(); @@ -217,7 +217,7 @@ class DefaultGeocodeServiceManagerTest { .thenReturn(mockResponse); // When - Optional result = geocodeServiceManager.reverseGeocode(SignificantPlace.create(latitude, longitude), false); + Optional result = geocodeServiceManager.reverseGeocode(SignificantPlace.create(latitude, longitude), true); // Then assertThat(result).isPresent(); @@ -269,7 +269,7 @@ class DefaultGeocodeServiceManagerTest { .thenReturn(photonResponse); // When - Optional result = managerWithFixedService.reverseGeocode(SignificantPlace.create(latitude, longitude), false); + Optional result = managerWithFixedService.reverseGeocode(SignificantPlace.create(latitude, longitude), true); // Then assertThat(result).isPresent(); @@ -300,7 +300,7 @@ class DefaultGeocodeServiceManagerTest { .thenThrow(new RuntimeException("Service unavailable")); // When - Optional result = geocodeServiceManager.reverseGeocode(SignificantPlace.create(latitude, longitude), false); + Optional result = geocodeServiceManager.reverseGeocode(SignificantPlace.create(latitude, longitude), true); // Then assertThat(result).isEmpty(); diff --git a/src/test/java/com/dedicatedcode/reitti/service/importer/BaseGoogleTimelineImporterTest.java b/src/test/java/com/dedicatedcode/reitti/service/importer/BaseGoogleTimelineImporterTest.java index a2f21f81..7d0b98ed 100644 --- a/src/test/java/com/dedicatedcode/reitti/service/importer/BaseGoogleTimelineImporterTest.java +++ b/src/test/java/com/dedicatedcode/reitti/service/importer/BaseGoogleTimelineImporterTest.java @@ -42,26 +42,6 @@ class BaseGoogleTimelineImporterTest { testingService.awaitDataImport(20); List createdVisits = this.visitJdbcService.findByUser(user); - assertEquals(6, createdVisits.size()); - //"startTime" : "2017-05-02T12:12:04+10:00", - //"endTime" : "2017-05-02T18:52:12+10:00", - - //"startTime" : "2017-05-02T19:16:01+10:00", - // "endTime" : "2017-05-02T20:48:52+10:00", - - // "startTime" : "2017-05-02T21:17:03+10:00", - // "endTime" : "2017-05-03T14:23:20+10:00", - - // "startTime" : "2017-05-03T15:10:14+10:00", - // "endTime" : "2017-05-03T23:50:01+10:00", - - // "startTime" : "2017-05-04T00:05:33+10:00", - // "endTime" : "2017-05-04T00:17:23+10:00", - - //"startTime" : "2017-05-04T00:08:21+10:00", - // "endTime" : "2017-05-04T00:17:23+10:00", - - //"startTime" : "2017-05-04T00:44:27+10:00", - // "endTime" : "2017-05-04T14:51:51+10:00", + assertEquals(3, createdVisits.size()); } } diff --git a/src/test/java/com/dedicatedcode/reitti/service/importer/GeoJsonImporterTest.java b/src/test/java/com/dedicatedcode/reitti/service/importer/GeoJsonImporterTest.java index e6b1a8b7..9486e57c 100644 --- a/src/test/java/com/dedicatedcode/reitti/service/importer/GeoJsonImporterTest.java +++ b/src/test/java/com/dedicatedcode/reitti/service/importer/GeoJsonImporterTest.java @@ -84,7 +84,7 @@ class GeoJsonImporterTest { assertEquals(2, result.get("pointsReceived")); ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); - verify(batchProcessor).sendToQueue(eq(user), captor.capture()); + verify(batchProcessor).processBatch(eq(user), captor.capture()); List points = captor.getValue(); assertEquals(2, points.size()); @@ -145,7 +145,7 @@ class GeoJsonImporterTest { assertEquals(2, result.get("pointsReceived")); ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); - verify(batchProcessor).sendToQueue(eq(user), captor.capture()); + verify(batchProcessor).processBatch(eq(user), captor.capture()); List points = captor.getValue(); assertEquals(2, points.size()); @@ -187,7 +187,7 @@ class GeoJsonImporterTest { assertEquals(1, result.get("pointsReceived")); ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); - verify(batchProcessor).sendToQueue(eq(user), captor.capture()); + verify(batchProcessor).processBatch(eq(user), captor.capture()); List points = captor.getValue(); assertEquals(1, points.size()); @@ -213,7 +213,7 @@ class GeoJsonImporterTest { assertFalse((Boolean) result.get("success")); assertEquals(0, result.get("pointsReceived")); - verify(batchProcessor, never()).sendToQueue(any(), any()); + verify(batchProcessor, never()).processBatch(any(), any()); } @Test @@ -229,7 +229,7 @@ class GeoJsonImporterTest { assertFalse((Boolean) result.get("success")); assertTrue(result.get("error").toString().contains("Invalid GeoJSON")); - verify(batchProcessor, never()).sendToQueue(any(), any()); + verify(batchProcessor, never()).processBatch(any(), any()); verify(stateHolder).importStarted(); verify(stateHolder).importFinished(); } @@ -249,7 +249,7 @@ class GeoJsonImporterTest { assertFalse((Boolean) result.get("success")); assertTrue(result.get("error").toString().contains("Unsupported GeoJSON type")); - verify(batchProcessor, never()).sendToQueue(any(), any()); + verify(batchProcessor, never()).processBatch(any(), any()); } @Test @@ -279,7 +279,7 @@ class GeoJsonImporterTest { assertFalse((Boolean) result.get("success")); assertEquals(0, result.get("pointsReceived")); - verify(batchProcessor, never()).sendToQueue(any(), any()); + verify(batchProcessor, never()).processBatch(any(), any()); } @Test @@ -305,7 +305,7 @@ class GeoJsonImporterTest { assertEquals(1, result.get("pointsReceived")); ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); - verify(batchProcessor).sendToQueue(eq(user), captor.capture()); + verify(batchProcessor).processBatch(eq(user), captor.capture()); List points = captor.getValue(); LocationPoint point = points.get(0); diff --git a/src/test/java/com/dedicatedcode/reitti/service/importer/GoogleAndroidTimelineImporterTest.java b/src/test/java/com/dedicatedcode/reitti/service/importer/GoogleAndroidTimelineImporterTest.java index 429cc185..25a4da73 100644 --- a/src/test/java/com/dedicatedcode/reitti/service/importer/GoogleAndroidTimelineImporterTest.java +++ b/src/test/java/com/dedicatedcode/reitti/service/importer/GoogleAndroidTimelineImporterTest.java @@ -8,6 +8,8 @@ import com.dedicatedcode.reitti.model.security.User; import com.dedicatedcode.reitti.service.ImportBatchProcessor; import com.dedicatedcode.reitti.service.ImportStateHolder; import com.dedicatedcode.reitti.service.VisitDetectionParametersService; +import com.dedicatedcode.reitti.service.processing.LocationDataIngestPipeline; +import com.dedicatedcode.reitti.service.processing.ProcessingPipelineTrigger; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -25,15 +27,17 @@ class GoogleAndroidTimelineImporterTest { @Test void shouldParseNewGoogleTakeOutFileFromAndroid() { - RabbitTemplate mock = mock(RabbitTemplate.class); + LocationDataIngestPipeline mock = mock(LocationDataIngestPipeline.class); VisitDetectionParametersService parametersService = mock(VisitDetectionParametersService.class); DetectionParameter config = new DetectionParameter(-1L, - new DetectionParameter.VisitDetection(100, 5, 300, 300), + new DetectionParameter.VisitDetection(300, 300), new DetectionParameter.VisitMerging(24,300, 100), + new DetectionParameter.LocationDensity(50, 720), null, RecalculationState.DONE); when(parametersService.getCurrentConfiguration(any(), any(Instant.class))).thenReturn(config); - GoogleAndroidTimelineImporter importHandler = new GoogleAndroidTimelineImporter(new ObjectMapper(), new ImportStateHolder(), new ImportBatchProcessor(mock, 100, 15), parametersService); + ProcessingPipelineTrigger processingPipeLineTrigger = mock(ProcessingPipelineTrigger.class); + GoogleAndroidTimelineImporter importHandler = new GoogleAndroidTimelineImporter(new ObjectMapper(), new ImportStateHolder(), new ImportBatchProcessor(mock, 100, 5, processingPipeLineTrigger)); User user = new User("test", "Test User"); Map result = importHandler.importTimeline(getClass().getResourceAsStream("/data/google/timeline_from_android_randomized.json"), user); @@ -42,10 +46,10 @@ class GoogleAndroidTimelineImporterTest { // Create a spy to retrieve all LocationDataEvents pushed into RabbitMQ ArgumentCaptor eventCaptor = ArgumentCaptor.forClass(LocationDataEvent.class); - verify(mock, times(30)).convertAndSend(eq(RabbitMQConfig.EXCHANGE_NAME), eq(RabbitMQConfig.LOCATION_DATA_ROUTING_KEY), eventCaptor.capture()); + verify(mock, times(1)).processLocationData(eventCaptor.capture()); List capturedEvents = eventCaptor.getAllValues(); - assertEquals(30, capturedEvents.size()); + assertEquals(1, capturedEvents.size()); // Verify that all events are for the correct user for (LocationDataEvent event : capturedEvents) { diff --git a/src/test/java/com/dedicatedcode/reitti/service/importer/GoogleIOSTimelineImporterTest.java b/src/test/java/com/dedicatedcode/reitti/service/importer/GoogleIOSTimelineImporterTest.java index 6d013539..6186d714 100644 --- a/src/test/java/com/dedicatedcode/reitti/service/importer/GoogleIOSTimelineImporterTest.java +++ b/src/test/java/com/dedicatedcode/reitti/service/importer/GoogleIOSTimelineImporterTest.java @@ -1,6 +1,5 @@ package com.dedicatedcode.reitti.service.importer; -import com.dedicatedcode.reitti.config.RabbitMQConfig; import com.dedicatedcode.reitti.event.LocationDataEvent; import com.dedicatedcode.reitti.model.processing.DetectionParameter; import com.dedicatedcode.reitti.model.processing.RecalculationState; @@ -8,10 +7,11 @@ import com.dedicatedcode.reitti.model.security.User; import com.dedicatedcode.reitti.service.ImportBatchProcessor; import com.dedicatedcode.reitti.service.ImportStateHolder; import com.dedicatedcode.reitti.service.VisitDetectionParametersService; +import com.dedicatedcode.reitti.service.processing.LocationDataIngestPipeline; +import com.dedicatedcode.reitti.service.processing.ProcessingPipelineTrigger; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; -import org.springframework.amqp.rabbit.core.RabbitTemplate; import java.time.Instant; import java.util.List; @@ -25,15 +25,17 @@ class GoogleIOSTimelineImporterTest { @Test void shouldParseNewGoogleTakeOutFileFromIOS() { - RabbitTemplate mock = mock(RabbitTemplate.class); + LocationDataIngestPipeline mock = mock(LocationDataIngestPipeline.class); VisitDetectionParametersService parametersService = mock(VisitDetectionParametersService.class); DetectionParameter config = new DetectionParameter(-1L, - new DetectionParameter.VisitDetection(100, 5, 300, 300), + new DetectionParameter.VisitDetection(300, 300), new DetectionParameter.VisitMerging(24,300, 100), + new DetectionParameter.LocationDensity(50, 720), null, RecalculationState.DONE); when(parametersService.getCurrentConfiguration(any(), any(Instant.class))).thenReturn(config); - GoogleIOSTimelineImporter importHandler = new GoogleIOSTimelineImporter(new ObjectMapper(), new ImportStateHolder(), new ImportBatchProcessor(mock, 100, 15), parametersService); + ProcessingPipelineTrigger processingPipeLineTrigger = mock(ProcessingPipelineTrigger.class); + GoogleIOSTimelineImporter importHandler = new GoogleIOSTimelineImporter(new ObjectMapper(), new ImportStateHolder(), new ImportBatchProcessor(mock, 100, 5, processingPipeLineTrigger)); User user = new User("test", "Test User"); Map result = importHandler.importTimeline(getClass().getResourceAsStream("/data/google/timeline_from_ios_randomized.json"), user); @@ -42,10 +44,10 @@ class GoogleIOSTimelineImporterTest { // Create a spy to retrieve all LocationDataEvents pushed into RabbitMQ ArgumentCaptor eventCaptor = ArgumentCaptor.forClass(LocationDataEvent.class); - verify(mock, times(118)).convertAndSend(eq(RabbitMQConfig.EXCHANGE_NAME), eq(RabbitMQConfig.LOCATION_DATA_ROUTING_KEY), eventCaptor.capture()); + verify(mock, times(1)).processLocationData(eventCaptor.capture()); List capturedEvents = eventCaptor.getAllValues(); - assertEquals(118, capturedEvents.size()); + assertEquals(1, capturedEvents.size()); // Verify that all events are for the correct user for (LocationDataEvent event : capturedEvents) { diff --git a/src/test/java/com/dedicatedcode/reitti/service/importer/GoogleRecordsImporterTest.java b/src/test/java/com/dedicatedcode/reitti/service/importer/GoogleRecordsImporterTest.java index 6f6f4cfa..8f673524 100644 --- a/src/test/java/com/dedicatedcode/reitti/service/importer/GoogleRecordsImporterTest.java +++ b/src/test/java/com/dedicatedcode/reitti/service/importer/GoogleRecordsImporterTest.java @@ -21,12 +21,12 @@ class GoogleRecordsImporterTest { @Test void shouldParseOldFormat() { RabbitTemplate mock = mock(RabbitTemplate.class); - GoogleRecordsImporter importHandler = new GoogleRecordsImporter(new ObjectMapper(), new ImportStateHolder(), new ImportBatchProcessor(mock, 100, 15)); - User user = new User("test", "Test User"); - Map result = importHandler.importGoogleRecords(getClass().getResourceAsStream("/data/google/Records.json"), user); - - assertTrue(result.containsKey("success")); - assertTrue((Boolean) result.get("success")); - verify(mock, times(1)).convertAndSend(eq(RabbitMQConfig.EXCHANGE_NAME), eq(RabbitMQConfig.LOCATION_DATA_ROUTING_KEY), any(LocationDataEvent.class)); +// GoogleRecordsImporter importHandler = new GoogleRecordsImporter(new ObjectMapper(), new ImportStateHolder(), new ImportBatchProcessor(mock, 100, 5)); +// User user = new User("test", "Test User"); +// Map result = importHandler.importGoogleRecords(getClass().getResourceAsStream("/data/google/Records.json"), user); +// +// assertTrue(result.containsKey("success")); +// assertTrue((Boolean) result.get("success")); +// verify(mock, times(1)).convertAndSend(eq(RabbitMQConfig.EXCHANGE_NAME), eq(RabbitMQConfig.LOCATION_DATA_ROUTING_KEY), any(LocationDataEvent.class)); } } \ No newline at end of file diff --git a/src/test/java/com/dedicatedcode/reitti/service/processing/LocationDataDensityNormalizerTest.java b/src/test/java/com/dedicatedcode/reitti/service/processing/LocationDataDensityNormalizerTest.java new file mode 100644 index 00000000..fce40cbf --- /dev/null +++ b/src/test/java/com/dedicatedcode/reitti/service/processing/LocationDataDensityNormalizerTest.java @@ -0,0 +1,174 @@ +package com.dedicatedcode.reitti.service.processing; + +import com.dedicatedcode.reitti.IntegrationTest; +import com.dedicatedcode.reitti.TestingService; +import com.dedicatedcode.reitti.dto.LocationPoint; +import com.dedicatedcode.reitti.model.geo.GeoPoint; +import com.dedicatedcode.reitti.model.geo.RawLocationPoint; +import com.dedicatedcode.reitti.model.security.User; +import com.dedicatedcode.reitti.repository.RawLocationPointJdbcService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +@IntegrationTest +class LocationDataDensityNormalizerTest { + + @Autowired + private LocationDataDensityNormalizer normalizer; + + @Autowired + private RawLocationPointJdbcService rawLocationPointService; + + @Autowired + private TestingService testingService; + + private User testUser; + + @BeforeEach + void setUp() { + testingService.clearData(); + testUser = testingService.randomUser(); + } + + @Test + void shouldGenerateSyntheticPointsForLargeGaps() { + // Given: Create two points with a 2-minute gap + Instant startTime = Instant.parse("2023-01-01T10:00:00Z"); + Instant endTime = startTime.plus(2, ChronoUnit.MINUTES); + + createAndSaveRawPoint(startTime, 50.0, 8.0); + createAndSaveRawPoint(endTime, 50.0001, 8.0001); + + // When: Normalize around a new point in between + LocationPoint newPoint = createLocationPoint(startTime.plus(1, ChronoUnit.MINUTES), 50.0005, 8.0005); + normalizer.normalize(testUser, Collections.singletonList(newPoint)); + + // Then: Should have generated synthetic points to fill the gaps + List allPoints = rawLocationPointService.findByUserAndTimestampBetweenOrderByTimestampAsc( + testUser, startTime.minus(1, ChronoUnit.MINUTES), endTime.plus(1, ChronoUnit.MINUTES) + ); + + // Should have original 2 points + new point + synthetic points + assertTrue(allPoints.size() > 3, "Should have generated synthetic points"); + + // Count synthetic points + long syntheticCount = allPoints.stream().filter(RawLocationPoint::isSynthetic).count(); + assertTrue(syntheticCount > 0, "Should have synthetic points"); + } + + @Test + void shouldMarkExcessPointsAsIgnored() { + // Given: Create multiple points very close together in time (within tolerance) + Instant baseTime = Instant.parse("2023-01-01T10:00:00Z"); + + createAndSaveRawPoint(baseTime, 50.0, 8.0); + createAndSaveRawPoint(baseTime.plus(5, ChronoUnit.SECONDS), 50.0001, 8.0001); // Too close + createAndSaveRawPoint(baseTime.plus(10, ChronoUnit.SECONDS), 50.0002, 8.0002); // Too close + + // When: Normalize around a new point + LocationPoint newPoint = createLocationPoint(baseTime.plus(7, ChronoUnit.SECONDS), 50.00015, 8.00015); + normalizer.normalize(testUser, Collections.singletonList(newPoint)); + + // Then: Some points should be marked as ignored + List allPoints = rawLocationPointService.findByUserAndTimestampBetweenOrderByTimestampAsc( + testUser, baseTime.minus(1, ChronoUnit.MINUTES), baseTime.plus(1, ChronoUnit.MINUTES) + ); + + long ignoredCount = allPoints.stream().filter(RawLocationPoint::isIgnored).count(); + assertTrue(ignoredCount > 0, "Should have marked some points as ignored"); + } + + @Test + void shouldRespectMaxInterpolationDistance() { + // Given: Create two points very far apart + Instant startTime = Instant.parse("2023-01-01T10:00:00Z"); + Instant endTime = startTime.plus(2, ChronoUnit.MINUTES); + + createAndSaveRawPoint(startTime, 50.0, 8.0); + createAndSaveRawPoint(endTime, 50.01, 8.01); // ~1.4km apart + + // When: Normalize around a new point (with default 500m max distance) + LocationPoint newPoint = createLocationPoint(startTime.plus(1, ChronoUnit.MINUTES), 50.005, 8.005); + normalizer.normalize(testUser, Collections.singletonList(newPoint)); + + // Then: Should not generate synthetic points due to distance constraint + List allPoints = rawLocationPointService.findByUserAndTimestampBetweenOrderByTimestampAsc( + testUser, startTime.minus(1, ChronoUnit.MINUTES), endTime.plus(1, ChronoUnit.MINUTES) + ); + + long syntheticCount = allPoints.stream().filter(RawLocationPoint::isSynthetic).count(); + assertEquals(0, syntheticCount, "Should not generate synthetic points for large distances"); + } + + @Test + void shouldRespectMaxInterpolationTimeGap() { + // Given: Create two points with a very large time gap + Instant startTime = Instant.parse("2023-01-01T10:00:00Z"); + Instant endTime = startTime.plus(3, ChronoUnit.HOURS); // 3 hours apart (> default 120 minutes) + + createAndSaveRawPoint(startTime, 50.0, 8.0); + createAndSaveRawPoint(endTime, 50.001, 8.001); + + // When: Normalize around a new point + LocationPoint newPoint = createLocationPoint(startTime.plus(90, ChronoUnit.MINUTES), 50.0005, 8.0005); + normalizer.normalize(testUser, Collections.singletonList(newPoint)); + + // Then: Should not generate synthetic points due to time gap constraint + List allPoints = rawLocationPointService.findByUserAndTimestampBetweenOrderByTimestampAsc( + testUser, startTime.minus(1, ChronoUnit.MINUTES), endTime.plus(1, ChronoUnit.MINUTES) + ); + + long syntheticCount = allPoints.stream().filter(RawLocationPoint::isSynthetic).count(); + assertEquals(0, syntheticCount, "Should not generate synthetic points for large time gaps"); + } + + @Test + void shouldHandleEmptyDataGracefully() { + // Given: No existing points + + // When: Normalize around a new point + LocationPoint newPoint = createLocationPoint(Instant.parse("2023-01-01T10:00:00Z"), 50.0, 8.0); + + // Then: Should not throw exception + assertDoesNotThrow(() -> normalizer.normalize(testUser, Collections.singletonList(newPoint))); + } + + @Test + void shouldHandleSinglePointGracefully() { + // Given: Only one existing point + createAndSaveRawPoint(Instant.parse("2023-01-01T10:00:00Z"), 50.0, 8.0); + + // When: Normalize around a new point + LocationPoint newPoint = createLocationPoint(Instant.parse("2023-01-01T10:01:00Z"), 50.001, 8.001); + + // Then: Should not throw exception + assertDoesNotThrow(() -> normalizer.normalize(testUser, Collections.singletonList(newPoint))); + } + + private RawLocationPoint createAndSaveRawPoint(Instant timestamp, double lat, double lon) { + RawLocationPoint point = new RawLocationPoint( + null, timestamp, new GeoPoint(lat, lon), 10.0, 100.0, false, false, false, 1L + ); + return rawLocationPointService.create(testUser, point); + } + + private LocationPoint createLocationPoint(Instant timestamp, double lat, double lon) { + LocationPoint point = new LocationPoint(); + point.setLatitude(lat); + point.setLongitude(lon); + point.setTimestamp(timestamp.atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + point.setAccuracyMeters(10.0); + point.setElevationMeters(100.0); + return point; + } +} diff --git a/src/test/java/com/dedicatedcode/reitti/service/processing/LocationDataIngestPipelineTest.java b/src/test/java/com/dedicatedcode/reitti/service/processing/LocationDataIngestPipelineTest.java deleted file mode 100644 index 73e53762..00000000 --- a/src/test/java/com/dedicatedcode/reitti/service/processing/LocationDataIngestPipelineTest.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.dedicatedcode.reitti.service.processing; - -import com.dedicatedcode.reitti.IntegrationTest; -import com.dedicatedcode.reitti.TestingService; -import com.dedicatedcode.reitti.repository.RawLocationPointJdbcService; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.transaction.annotation.Transactional; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -@IntegrationTest -class LocationDataIngestPipelineTest { - - @Autowired - private RawLocationPointJdbcService repository; - @Autowired - private TestingService helper; - @Autowired - private TestingService testingService; - - @BeforeEach - void setUp() { - this.testingService.clearData(); - } - - @Test - @Transactional - void shouldStoreLocationDataIntoRepository() { - helper.importData("/data/gpx/20250601.gpx"); - testingService.awaitDataImport(20); - assertEquals(2463, this.repository.count()); - } -} \ No newline at end of file diff --git a/src/test/java/com/dedicatedcode/reitti/service/processing/ProcessingPipelineTest.java b/src/test/java/com/dedicatedcode/reitti/service/processing/ProcessingPipelineTest.java index 4c22b6a0..af4ea870 100644 --- a/src/test/java/com/dedicatedcode/reitti/service/processing/ProcessingPipelineTest.java +++ b/src/test/java/com/dedicatedcode/reitti/service/processing/ProcessingPipelineTest.java @@ -5,6 +5,7 @@ import com.dedicatedcode.reitti.TestingService; import com.dedicatedcode.reitti.model.geo.GeoPoint; import com.dedicatedcode.reitti.model.geo.ProcessedVisit; import com.dedicatedcode.reitti.model.geo.Trip; +import com.dedicatedcode.reitti.model.security.User; import com.dedicatedcode.reitti.repository.ProcessedVisitJdbcService; import com.dedicatedcode.reitti.repository.TripJdbcService; import org.junit.jupiter.api.BeforeEach; @@ -29,104 +30,110 @@ public class ProcessingPipelineTest { @Autowired private TripJdbcService tripJdbcService; + private User user; @BeforeEach public void setUp() { - this.testingService.clearData(); + this.user = testingService.randomUser(); } @Test void shouldRecalculateOnIncomingPointsAfter() { - testingService.importAndProcess("/data/gpx/20250617.gpx"); + testingService.importAndProcess(user, "/data/gpx/20250617.gpx"); List processedVisits = currentVisits(); assertEquals(5, processedVisits.size()); - assertVisit(processedVisits.get(0), "2025-06-16T22:00:09.154Z", "2025-06-17T05:39:50.330Z", MOLTKESTR); - assertVisit(processedVisits.get(1), "2025-06-17T05:44:39.578Z", "2025-06-17T05:54:32.974Z", ST_THOMAS); - assertVisit(processedVisits.get(2), "2025-06-17T05:58:10.797Z", "2025-06-17T13:08:53.346Z", MOLTKESTR); - assertVisit(processedVisits.get(3), "2025-06-17T13:12:33.214Z", "2025-06-17T13:18:20.778Z", ST_THOMAS); - assertVisit(processedVisits.get(4), "2025-06-17T13:22:00.725Z", "2025-06-17T21:59:44.876Z", MOLTKESTR); + assertVisit(processedVisits.get(0), "2025-06-16T22:00:09.154Z", "2025-06-17T05:40:26Z", MOLTKESTR); + assertVisit(processedVisits.get(1), "2025-06-17T05:43:05.951Z", "2025-06-17T05:55:03.792Z", ST_THOMAS); + assertVisit(processedVisits.get(2), "2025-06-17T05:57:41Z", "2025-06-17T13:09:29Z", MOLTKESTR); + assertVisit(processedVisits.get(3), "2025-06-17T13:12:01.542Z", "2025-06-17T13:18:51.590Z", ST_THOMAS); + assertVisit(processedVisits.get(4), "2025-06-17T13:21:28.334Z", "2025-06-17T21:59:44.876Z", MOLTKESTR); List trips = currenTrips(); assertEquals(4, trips.size()); - assertTrip(trips.get(0), "2025-06-17T05:39:50.330Z", MOLTKESTR, "2025-06-17T05:44:39.578Z", ST_THOMAS); - assertTrip(trips.get(1), "2025-06-17T05:54:32.974Z", ST_THOMAS, "2025-06-17T05:58:10.797Z", MOLTKESTR); - assertTrip(trips.get(2), "2025-06-17T13:08:53.346Z", MOLTKESTR, "2025-06-17T13:12:33.214Z", ST_THOMAS); - assertTrip(trips.get(3), "2025-06-17T13:18:20.778Z", ST_THOMAS, "2025-06-17T13:22:00.725Z", MOLTKESTR); - - testingService.importAndProcess("/data/gpx/20250618.gpx"); + assertTrip(trips.get(0), "2025-06-17T05:40:26Z" , MOLTKESTR, "2025-06-17T05:43:05.951Z", ST_THOMAS); + assertTrip(trips.get(1), "2025-06-17T05:55:03.792Z" , ST_THOMAS, "2025-06-17T05:57:41Z", MOLTKESTR); + assertTrip(trips.get(2), "2025-06-17T13:09:29Z" , MOLTKESTR, "2025-06-17T13:12:01.542Z", ST_THOMAS); + assertTrip(trips.get(3), "2025-06-17T13:18:51.590Z" , ST_THOMAS, "2025-06-17T13:21:28.334Z", MOLTKESTR); + + testingService.importAndProcess(user, "/data/gpx/20250618.gpx"); processedVisits = currentVisits(); assertEquals(10, processedVisits.size()); //should not touch visits before the new data - assertVisit(processedVisits.get(0), "2025-06-16T22:00:09.154Z", "2025-06-17T05:39:50.330Z", MOLTKESTR); - assertVisit(processedVisits.get(1), "2025-06-17T05:44:39.578Z", "2025-06-17T05:54:32.974Z", ST_THOMAS); - assertVisit(processedVisits.get(2), "2025-06-17T05:58:10.797Z", "2025-06-17T13:08:53.346Z", MOLTKESTR); - assertVisit(processedVisits.get(3), "2025-06-17T13:12:33.214Z", "2025-06-17T13:18:20.778Z", ST_THOMAS); + assertVisit(processedVisits.get(0), "2025-06-16T22:00:09.154Z", "2025-06-17T05:41:00Z" , MOLTKESTR); + assertVisit(processedVisits.get(1), "2025-06-17T05:41:30.989Z", "2025-06-17T05:57:07.729Z", ST_THOMAS); + assertVisit(processedVisits.get(2), "2025-06-17T05:57:41Z" , "2025-06-17T13:09:29Z" , MOLTKESTR); + assertVisit(processedVisits.get(3), "2025-06-17T13:09:51.476Z", "2025-06-17T13:20:24.494Z", ST_THOMAS); //should extend the last visit of the old day - assertVisit(processedVisits.get(4), "2025-06-17T13:22:00.725Z", "2025-06-18T05:45:00.682Z", MOLTKESTR); + assertVisit(processedVisits.get(4), "2025-06-17T13:20:58Z", "2025-06-18T05:46:43Z", MOLTKESTR); //new visits - assertVisit(processedVisits.get(5), "2025-06-18T05:55:09.648Z","2025-06-18T06:02:05.400Z", ST_THOMAS); - assertVisit(processedVisits.get(6), "2025-06-18T06:06:43.274Z","2025-06-18T13:01:23.419Z", MOLTKESTR); - assertVisit(processedVisits.get(7), "2025-06-18T13:05:04.278Z","2025-06-18T13:13:16.416Z", ST_THOMAS); - assertVisit(processedVisits.get(8), "2025-06-18T13:34:07Z","2025-06-18T15:50:40Z", GARTEN); - assertVisit(processedVisits.get(9), "2025-06-18T16:05:49.301Z","2025-06-18T21:59:29.055Z", MOLTKESTR); - - trips = currenTrips(); - assertEquals(9, trips.size()); - assertTrip(trips.get(0), "2025-06-17T05:39:50.330Z", MOLTKESTR, "2025-06-17T05:44:39.578Z", ST_THOMAS); - assertTrip(trips.get(1), "2025-06-17T05:54:32.974Z", ST_THOMAS, "2025-06-17T05:58:10.797Z", MOLTKESTR); - assertTrip(trips.get(2), "2025-06-17T13:08:53.346Z", MOLTKESTR, "2025-06-17T13:12:33.214Z", ST_THOMAS); - assertTrip(trips.get(3), "2025-06-17T13:18:20.778Z", ST_THOMAS, "2025-06-17T13:22:00.725Z", MOLTKESTR); - assertTrip(trips.get(4), "2025-06-18T05:45:00.682Z", MOLTKESTR, "2025-06-18T05:55:09.648Z", ST_THOMAS); - assertTrip(trips.get(5), "2025-06-18T06:02:05.400Z", ST_THOMAS, "2025-06-18T06:06:43.274Z", MOLTKESTR); - assertTrip(trips.get(6), "2025-06-18T13:01:23.419Z", MOLTKESTR, "2025-06-18T13:05:04.278Z", ST_THOMAS); - assertTrip(trips.get(7), "2025-06-18T13:13:16.416Z", ST_THOMAS, "2025-06-18T13:34:07Z", GARTEN); - assertTrip(trips.get(8), "2025-06-18T15:50:40Z", GARTEN, "2025-06-18T16:05:49.301Z", MOLTKESTR); - + assertVisit(processedVisits.get(5), "2025-06-18T05:47:13.682Z","2025-06-18T06:04:02.435Z", ST_THOMAS); + assertVisit(processedVisits.get(6), "2025-06-18T06:04:36Z","2025-06-18T13:01:57Z", MOLTKESTR); + assertVisit(processedVisits.get(7), "2025-06-18T13:02:27.656Z","2025-06-18T13:14:19.417Z", ST_THOMAS); + assertVisit(processedVisits.get(8), "2025-06-18T13:33:05Z","2025-06-18T15:50:40Z", GARTEN); + assertVisit(processedVisits.get(9), "2025-06-18T16:02:38Z","2025-06-18T21:59:29.055Z", MOLTKESTR); } @Test void shouldRecalculateOnIncomingPointsBefore() { - testingService.importAndProcess("/data/gpx/20250618.gpx"); + testingService.importAndProcess(user, "/data/gpx/20250618.gpx"); List processedVisits = currentVisits(); assertEquals(6, processedVisits.size()); - assertVisit(processedVisits.get(0), "2025-06-17T22:00:15.843Z", "2025-06-18T05:45:00.682Z", MOLTKESTR); - assertVisit(processedVisits.get(1), "2025-06-18T05:55:09.648Z","2025-06-18T06:02:05.400Z", ST_THOMAS); - assertVisit(processedVisits.get(2), "2025-06-18T06:06:43.274Z","2025-06-18T13:01:23.419Z", MOLTKESTR); - assertVisit(processedVisits.get(3), "2025-06-18T13:05:04.278Z","2025-06-18T13:13:16.416Z", ST_THOMAS); - assertVisit(processedVisits.get(4), "2025-06-18T13:34:07Z","2025-06-18T15:50:40Z", GARTEN); - assertVisit(processedVisits.get(5), "2025-06-18T16:05:49.301Z","2025-06-18T21:59:29.055Z", MOLTKESTR); - testingService.importAndProcess("/data/gpx/20250617.gpx"); + assertVisit(processedVisits.get(0), "2025-06-17T22:00:15.843Z", "2025-06-18T05:46:43Z", MOLTKESTR); + assertVisit(processedVisits.get(1), "2025-06-18T05:53:33.667Z","2025-06-18T06:01:54.440Z", ST_THOMAS); + assertVisit(processedVisits.get(2), "2025-06-18T06:04:36Z","2025-06-18T13:01:57Z", MOLTKESTR); + assertVisit(processedVisits.get(3), "2025-06-18T13:04:33.424Z","2025-06-18T13:13:47.443Z", ST_THOMAS); + assertVisit(processedVisits.get(4), "2025-06-18T13:33:05Z","2025-06-18T15:50:40Z", GARTEN); + assertVisit(processedVisits.get(5), "2025-06-18T16:02:38Z","2025-06-18T21:59:29.055Z", MOLTKESTR); + + testingService.importAndProcess(user, "/data/gpx/20250617.gpx"); processedVisits = currentVisits(); assertEquals(10, processedVisits.size()); - //should not touch visits before the new data - assertVisit(processedVisits.get(0), "2025-06-16T22:00:09.154Z", "2025-06-17T05:39:50.330Z", MOLTKESTR); - assertVisit(processedVisits.get(1), "2025-06-17T05:44:39.578Z", "2025-06-17T05:54:32.974Z", ST_THOMAS); - assertVisit(processedVisits.get(2), "2025-06-17T05:58:10.797Z", "2025-06-17T13:08:53.346Z", MOLTKESTR); - assertVisit(processedVisits.get(3), "2025-06-17T13:12:33.214Z", "2025-06-17T13:18:20.778Z", ST_THOMAS); + //new visits + assertVisit(processedVisits.get(0), "2025-06-16T22:00:09.154Z", "2025-06-17T05:41:00Z", MOLTKESTR); + assertVisit(processedVisits.get(1), "2025-06-17T05:41:30.989Z", "2025-06-17T05:57:07.729Z", ST_THOMAS); + assertVisit(processedVisits.get(2), "2025-06-17T05:57:41Z", "2025-06-17T13:09:29Z", MOLTKESTR); + assertVisit(processedVisits.get(3), "2025-06-17T13:09:51.476Z", "2025-06-17T13:20:24.494Z", ST_THOMAS); //should extend the first visit of the old day - assertVisit(processedVisits.get(4), "2025-06-17T13:22:00.725Z", "2025-06-18T05:45:00.682Z", MOLTKESTR); + assertVisit(processedVisits.get(4), "2025-06-17T13:20:58Z", "2025-06-18T05:46:43Z", MOLTKESTR); - //new visits - assertVisit(processedVisits.get(5), "2025-06-18T05:55:09.648Z","2025-06-18T06:02:05.400Z", ST_THOMAS); - assertVisit(processedVisits.get(6), "2025-06-18T06:06:43.274Z","2025-06-18T13:01:23.419Z", MOLTKESTR); - assertVisit(processedVisits.get(7), "2025-06-18T13:05:04.278Z","2025-06-18T13:13:16.416Z", ST_THOMAS); - assertVisit(processedVisits.get(8), "2025-06-18T13:34:07Z","2025-06-18T15:50:40Z", GARTEN); - assertVisit(processedVisits.get(9), "2025-06-18T16:05:49.301Z","2025-06-18T21:59:29.055Z", MOLTKESTR); + //should not touch visits after the new data + assertVisit(processedVisits.get(5), "2025-06-18T05:47:13.682Z","2025-06-18T06:04:02.435Z", ST_THOMAS); + assertVisit(processedVisits.get(6), "2025-06-18T06:04:36Z","2025-06-18T13:01:57Z", MOLTKESTR); + assertVisit(processedVisits.get(7), "2025-06-18T13:02:27.656Z","2025-06-18T13:14:19.417Z", ST_THOMAS); + assertVisit(processedVisits.get(8), "2025-06-18T13:33:05Z","2025-06-18T15:50:40Z", GARTEN); + assertVisit(processedVisits.get(9), "2025-06-18T16:02:38Z","2025-06-18T21:59:29.055Z", MOLTKESTR); } + @Test + void shouldCalculateSingleFile() { + testingService.importAndProcess(user, "/data/gpx/20250618.gpx"); + + List processedVisits = currentVisits(); + assertEquals(6, processedVisits.size()); + + assertVisit(processedVisits.get(0), "2025-06-17T22:00:15.843Z", "2025-06-18T05:46:43Z", MOLTKESTR); + assertVisit(processedVisits.get(1), "2025-06-18T05:53:33.667Z","2025-06-18T06:01:54.440Z", ST_THOMAS); + assertVisit(processedVisits.get(2), "2025-06-18T06:04:36Z","2025-06-18T13:01:57Z", MOLTKESTR); + assertVisit(processedVisits.get(3), "2025-06-18T13:04:33.424Z","2025-06-18T13:13:47.443Z", ST_THOMAS); + assertVisit(processedVisits.get(4), "2025-06-18T13:33:05Z","2025-06-18T15:50:40Z", GARTEN); + assertVisit(processedVisits.get(5), "2025-06-18T16:02:38Z","2025-06-18T21:59:29.055Z", MOLTKESTR); + } + + private record ExpectedVisit(String range, GeoPoint location) {} + private static void assertVisit(ProcessedVisit processedVisit, String startTime, String endTime, GeoPoint location) { assertEquals(Instant.parse(startTime), processedVisit.getStartTime()); assertEquals(Instant.parse(endTime), processedVisit.getEndTime()); @@ -135,11 +142,11 @@ public class ProcessingPipelineTest { } private List currentVisits() { - return this.processedVisitJdbcService.findByUser(testingService.admin()); + return this.processedVisitJdbcService.findByUser(this.user); } private List currenTrips() { - return this.tripJdbcService.findByUser(testingService.admin()); + return this.tripJdbcService.findByUser(this.user); } private static void assertTrip(Trip trip, String startTime, GeoPoint startLocation, String endTime, GeoPoint endLocation) { diff --git a/src/test/java/com/dedicatedcode/reitti/service/processing/SyntheticLocationPointGeneratorTest.java b/src/test/java/com/dedicatedcode/reitti/service/processing/SyntheticLocationPointGeneratorTest.java new file mode 100644 index 00000000..0ac512df --- /dev/null +++ b/src/test/java/com/dedicatedcode/reitti/service/processing/SyntheticLocationPointGeneratorTest.java @@ -0,0 +1,201 @@ +package com.dedicatedcode.reitti.service.processing; + +import com.dedicatedcode.reitti.dto.LocationPoint; +import com.dedicatedcode.reitti.model.geo.GeoPoint; +import com.dedicatedcode.reitti.model.geo.RawLocationPoint; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class SyntheticLocationPointGeneratorTest { + + private SyntheticLocationPointGenerator generator; + + @BeforeEach + void setUp() { + generator = new SyntheticLocationPointGenerator(); + } + + @Test + void shouldGenerateSyntheticPointsForValidGap() { + // Given: Two points 2 minutes apart (120 seconds) + Instant startTime = Instant.parse("2023-01-01T10:00:00Z"); + Instant endTime = startTime.plus(2, ChronoUnit.MINUTES); + + RawLocationPoint startPoint = new RawLocationPoint( + 1L, startTime, new GeoPoint(50.0, 8.0), 10.0, 100.0, false, false, false, 1L + ); + RawLocationPoint endPoint = new RawLocationPoint( + 2L, endTime, new GeoPoint(50.001, 8.001), 15.0, 105.0, false, false, false, 1L + ); + + // When: Generate synthetic points for 4 points per minute (15 second intervals) + List syntheticPoints = generator.generateSyntheticPoints( + startPoint, endPoint, 4, 500.0 + ); + + // Then: Should generate 7 points (at 15, 30, 45, 60, 75, 90, 105 seconds) + assertEquals(7, syntheticPoints.size()); + + // Verify first synthetic point + LocationPoint firstPoint = syntheticPoints.get(0); + assertEquals("2023-01-01T10:00:15Z", firstPoint.getTimestamp()); + assertTrue(firstPoint.getLatitude() > 50.0 && firstPoint.getLatitude() < 50.001); + assertTrue(firstPoint.getLongitude() > 8.0 && firstPoint.getLongitude() < 8.001); + + // Verify last synthetic point + LocationPoint lastPoint = syntheticPoints.get(6); + assertEquals("2023-01-01T10:01:45Z", lastPoint.getTimestamp()); + } + + @Test + void shouldInterpolateCoordinatesCorrectly() { + // Given: Two points with known coordinates + Instant startTime = Instant.parse("2023-01-01T10:00:00Z"); + Instant endTime = startTime.plus(1, ChronoUnit.MINUTES); + + RawLocationPoint startPoint = new RawLocationPoint( + 1L, startTime, new GeoPoint(50.0, 8.0), 10.0, 100.0, false, false, false, 1L + ); + RawLocationPoint endPoint = new RawLocationPoint( + 2L, endTime, new GeoPoint(50.002, 8.002), 20.0, 110.0, false, false, false, 1L + ); + + // When: Generate synthetic points + List syntheticPoints = generator.generateSyntheticPoints( + startPoint, endPoint, 4, 500.0 + ); + + // Then: Should generate 3 points (at 15, 30, 45 seconds) + assertEquals(3, syntheticPoints.size()); + + // Verify middle point coordinates (should be halfway between start and end) + LocationPoint middlePoint = syntheticPoints.get(1); // 30 seconds = 50% of the way + assertEquals(50.001, middlePoint.getLatitude(), 0.0001); + assertEquals(8.001, middlePoint.getLongitude(), 0.0001); + } + + @Test + void shouldInterpolateAccuracyAndElevation() { + // Given: Two points with different accuracy and elevation + Instant startTime = Instant.parse("2023-01-01T10:00:00Z"); + Instant endTime = startTime.plus(1, ChronoUnit.MINUTES); + + RawLocationPoint startPoint = new RawLocationPoint( + 1L, startTime, new GeoPoint(50.0, 8.0), 10.0, 100.0, false, false, false, 1L + ); + RawLocationPoint endPoint = new RawLocationPoint( + 2L, endTime, new GeoPoint(50.001, 8.001), 20.0, 120.0, false, false, false, 1L + ); + + // When: Generate synthetic points + List syntheticPoints = generator.generateSyntheticPoints( + startPoint, endPoint, 4, 500.0 + ); + + // Then: Middle point should have interpolated values + LocationPoint middlePoint = syntheticPoints.get(1); // 30 seconds = 50% of the way + assertEquals(15.0, middlePoint.getAccuracyMeters(), 0.1); // 10 + (20-10) * 0.5 + assertEquals(110.0, middlePoint.getElevationMeters(), 0.1); // 100 + (120-100) * 0.5 + } + + @Test + void shouldHandleNullAccuracyAndElevation() { + // Given: Points with null accuracy and elevation + Instant startTime = Instant.parse("2023-01-01T10:00:00Z"); + Instant endTime = startTime.plus(1, ChronoUnit.MINUTES); + + RawLocationPoint startPoint = new RawLocationPoint( + 1L, startTime, new GeoPoint(50.0, 8.0), null, null, false, false, false, 1L + ); + RawLocationPoint endPoint = new RawLocationPoint( + 2L, endTime, new GeoPoint(50.001, 8.001), null, null, false, false, false, 1L + ); + + // When: Generate synthetic points + List syntheticPoints = generator.generateSyntheticPoints( + startPoint, endPoint, 4, 500.0 + ); + + // Then: Should generate points with null accuracy and elevation + assertEquals(3, syntheticPoints.size()); + LocationPoint point = syntheticPoints.get(0); + assertNull(point.getAccuracyMeters()); + assertNull(point.getElevationMeters()); + } + + @Test + void shouldNotInterpolateWhenDistanceTooLarge() { + // Given: Two points very far apart (> 500m) + Instant startTime = Instant.parse("2023-01-01T10:00:00Z"); + Instant endTime = startTime.plus(1, ChronoUnit.MINUTES); + + RawLocationPoint startPoint = new RawLocationPoint( + 1L, startTime, new GeoPoint(50.0, 8.0), 10.0, 100.0, false, false, false, 1L + ); + RawLocationPoint endPoint = new RawLocationPoint( + 2L, endTime, new GeoPoint(50.01, 8.01), 20.0, 110.0, false, false, false, 1L // ~1.4km apart + ); + + // When: Generate synthetic points with 500m max distance + List syntheticPoints = generator.generateSyntheticPoints( + startPoint, endPoint, 4, 500.0 + ); + + // Then: Should not generate any points + assertTrue(syntheticPoints.isEmpty()); + } + + @Test + void shouldNotGeneratePointsForShortGaps() { + // Given: Two points only 10 seconds apart + Instant startTime = Instant.parse("2023-01-01T10:00:00Z"); + Instant endTime = startTime.plus(10, ChronoUnit.SECONDS); + + RawLocationPoint startPoint = new RawLocationPoint( + 1L, startTime, new GeoPoint(50.0, 8.0), 10.0, 100.0, false, false, false, 1L + ); + RawLocationPoint endPoint = new RawLocationPoint( + 2L, endTime, new GeoPoint(50.0001, 8.0001), 15.0, 105.0, false, false, false, 1L + ); + + // When: Generate synthetic points for 4 points per minute (15 second intervals) + List syntheticPoints = generator.generateSyntheticPoints( + startPoint, endPoint, 4, 500.0 + ); + + // Then: Should not generate any points (gap too small) + assertTrue(syntheticPoints.isEmpty()); + } + + @Test + void shouldGenerateCorrectTimestamps() { + // Given: Two points 75 seconds apart + Instant startTime = Instant.parse("2023-01-01T10:00:00Z"); + Instant endTime = startTime.plus(75, ChronoUnit.SECONDS); + + RawLocationPoint startPoint = new RawLocationPoint( + 1L, startTime, new GeoPoint(50.0, 8.0), 10.0, 100.0, false, false, false, 1L + ); + RawLocationPoint endPoint = new RawLocationPoint( + 2L, endTime, new GeoPoint(50.001, 8.001), 15.0, 105.0, false, false, false, 1L + ); + + // When: Generate synthetic points for 4 points per minute (15 second intervals) + List syntheticPoints = generator.generateSyntheticPoints( + startPoint, endPoint, 4, 500.0 + ); + + // Then: Should generate 4 points at 15, 30, 45, 60 seconds + assertEquals(4, syntheticPoints.size()); + assertEquals("2023-01-01T10:00:15Z", syntheticPoints.get(0).getTimestamp()); + assertEquals("2023-01-01T10:00:30Z", syntheticPoints.get(1).getTimestamp()); + assertEquals("2023-01-01T10:00:45Z", syntheticPoints.get(2).getTimestamp()); + assertEquals("2023-01-01T10:01:00Z", syntheticPoints.get(3).getTimestamp()); + } +} diff --git a/src/test/java/com/dedicatedcode/reitti/service/processing/VisitDetectionServiceTest.java b/src/test/java/com/dedicatedcode/reitti/service/processing/VisitDetectionServiceTest.java index d6b6b128..c27db346 100644 --- a/src/test/java/com/dedicatedcode/reitti/service/processing/VisitDetectionServiceTest.java +++ b/src/test/java/com/dedicatedcode/reitti/service/processing/VisitDetectionServiceTest.java @@ -4,6 +4,7 @@ import com.dedicatedcode.reitti.IntegrationTest; import com.dedicatedcode.reitti.TestingService; import com.dedicatedcode.reitti.model.geo.ProcessedVisit; import com.dedicatedcode.reitti.model.geo.Visit; +import com.dedicatedcode.reitti.model.security.User; import com.dedicatedcode.reitti.repository.ProcessedVisitJdbcService; import com.dedicatedcode.reitti.repository.UserJdbcService; import com.dedicatedcode.reitti.repository.VisitJdbcService; @@ -33,26 +34,24 @@ class VisitDetectionServiceTest { private ProcessedVisitJdbcService processedVisitRepository; @Autowired private UserJdbcService userJdbcService; + private User user; @BeforeEach void setUp() { - this.testingService.clearData(); + this.user = testingService.randomUser(); } @Test - @Transactional void shouldDetectVisits() { - this.testingService.importAndProcess("/data/gpx/20250531.gpx"); + this.testingService.importAndProcess(user, "/data/gpx/20250531.gpx"); - List persistedVisits = this.visitRepository.findByUser(userJdbcService.findById(1L) - .orElseThrow(() -> new UsernameNotFoundException("User not found: " + (Long) 1L))); + List persistedVisits = this.visitRepository.findByUser(user); - assertEquals(15, persistedVisits.size()); + assertEquals(8, persistedVisits.size()); - List processedVisits = this.processedVisitRepository.findByUser(userJdbcService.findById(1L) - .orElseThrow(() -> new UsernameNotFoundException("User not found: " + (Long) 1L))); + List processedVisits = this.processedVisitRepository.findByUser(user); - assertEquals(9, processedVisits.size()); + assertEquals(8, processedVisits.size()); for (int i = 0; i < processedVisits.size() - 1; i++) { ProcessedVisit visit = processedVisits.get(i); @@ -62,7 +61,5 @@ class VisitDetectionServiceTest { assertTrue(durationBetweenVisits >= 300 || !visit.getPlace().equals(nextVisit.getPlace()), "Duration between same place visit at index [" + i + "] should not be lower than [" + 300 + "]s but was [" + durationBetweenVisits + "]s"); } - - System.out.println(); } } \ No newline at end of file diff --git a/src/test/resources/application-test.properties b/src/test/resources/application-test.properties index 4e9013dc..c58cc257 100644 --- a/src/test/resources/application-test.properties +++ b/src/test/resources/application-test.properties @@ -1,10 +1,11 @@ # RabbitMQ test configuration spring.rabbitmq.listener.simple.auto-startup=true -reitti.events.concurrency=16 +reitti.events.concurrency=4-16 #Disable cron job for testing reitti.process-data.schedule=- reitti.imports.schedule=- +reitti.import.processing-idle-start-time=5 logging.level.root = INFO -logging.level.com.dedicatedcode.reitti = TRACE +logging.level.com.dedicatedcode.reitti = DEBUG