From 2a07f7d3399dbdfa91a952fe55cfe08860ef08be Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Sun, 14 Dec 2025 15:23:04 +0100 Subject: [PATCH] 523 bug reitti misses some visits (#548) --- .../dedicatedcode/reitti/tools/GpxSender.java | 52 ++++++++++++------- .../settings/ManageDataController.java | 12 +++-- .../repository/UserSettingsJdbcService.java | 4 ++ .../service/DefaultImportProcessor.java | 13 +++-- .../GoogleAndroidTimelineImporter.java | 2 +- .../UnifiedLocationProcessingService.java | 4 +- 6 files changed, 56 insertions(+), 31 deletions(-) diff --git a/docs/tools/gpx-sender/src/main/java/com/dedicatedcode/reitti/tools/GpxSender.java b/docs/tools/gpx-sender/src/main/java/com/dedicatedcode/reitti/tools/GpxSender.java index 6753a5f6..fa63dcc4 100644 --- a/docs/tools/gpx-sender/src/main/java/com/dedicatedcode/reitti/tools/GpxSender.java +++ b/docs/tools/gpx-sender/src/main/java/com/dedicatedcode/reitti/tools/GpxSender.java @@ -48,15 +48,17 @@ public class GpxSender { public static void main(String[] args) { if (args.length < 1) { - System.err.println("Usage: java -jar gpx-sender.jar --url --token [--interval ]"); + System.err.println("Usage: java -jar gpx-sender.jar --url --token [--interval ] [--original-time]"); System.err.println("Example: java -jar gpx-sender.jar track.gpx --url http://localhost:8080 --token your-api-token --interval 15"); + System.err.println(" java -jar gpx-sender.jar track.gpx --url http://localhost:8080 --token your-api-token --original-time"); System.exit(1); } String gpxFile = args[0]; String reittiUrl = null; String apiToken = null; - int intervalSeconds = 15; + double intervalSeconds = 15.0; + boolean useOriginalTime = false; // Parse named parameters for (int i = 1; i < args.length; i++) { @@ -73,9 +75,13 @@ public class GpxSender { break; case "--interval": if (i + 1 < args.length) { - intervalSeconds = Integer.parseInt(args[++i]); + intervalSeconds = Double.parseDouble(args[++i]); } break; + case "--original-time": + case "-ot": + useOriginalTime = true; + break; default: System.err.println("Unknown parameter: " + args[i]); System.exit(1); @@ -96,9 +102,13 @@ public class GpxSender { System.out.println("Loaded " + trackPoints.size() + " track points from " + gpxFile); System.out.println("Sending to: " + reittiUrl); - System.out.println("Interval: " + intervalSeconds + " seconds"); + if (useOriginalTime) { + System.out.println("Using original timestamps from GPX file"); + } else { + System.out.println("Interval: " + intervalSeconds + " seconds"); + } - sendTrackPoints(trackPoints, reittiUrl, apiToken, intervalSeconds); + sendTrackPoints(trackPoints, reittiUrl, apiToken, intervalSeconds, useOriginalTime); } catch (Exception e) { System.err.println("Error: " + e.getMessage()); @@ -135,7 +145,7 @@ public class GpxSender { return trackPoints; } - private static void sendTrackPoints(List trackPoints, String reittiUrl, String apiToken, int intervalSeconds) throws Exception { + private static void sendTrackPoints(List trackPoints, String reittiUrl, String apiToken, double intervalSeconds, boolean useOriginalTime) throws Exception { ObjectMapper objectMapper = new ObjectMapper(); objectMapper.registerModule(new JavaTimeModule()); @@ -147,19 +157,25 @@ public class GpxSender { for (int i = 0; i < trackPoints.size(); i++) { TrackPoint point = trackPoints.get(i); - // Calculate adjusted timestamp - start from now and preserve original intervals + // Calculate timestamp based on mode Instant adjustedTime; - if (i == 0) { - // First point gets current time - adjustedTime = startTime; - } else if (i > 0 && trackPoints.get(i-1).timestamp != null && point.timestamp != null) { - // Calculate time difference from previous point and add to previous adjusted time - TrackPoint prevPoint = trackPoints.get(i-1); - long durationFromPrev = point.timestamp.getEpochSecond() - prevPoint.timestamp.getEpochSecond(); - adjustedTime = startTime.plusSeconds((long) i * intervalSeconds); + if (useOriginalTime) { + // Use original timestamp from GPX file + adjustedTime = point.timestamp != null ? point.timestamp : startTime.plusMillis((long) (i * intervalSeconds * 1000)); } else { - // Fallback: distribute points evenly from start time - adjustedTime = startTime.plusSeconds((long) i * intervalSeconds); + // Calculate adjusted timestamp - start from now and preserve original intervals + if (i == 0) { + // First point gets current time + adjustedTime = startTime; + } else if (trackPoints.get(i - 1).timestamp != null && point.timestamp != null) { + // Calculate time difference from previous point and add to previous adjusted time + TrackPoint prevPoint = trackPoints.get(i-1); + long durationFromPrev = point.timestamp.getEpochSecond() - prevPoint.timestamp.getEpochSecond(); + adjustedTime = startTime.plusMillis((long) (i * intervalSeconds * 1000)); + } else { + // Fallback: distribute points evenly from start time + adjustedTime = startTime.plusMillis((long) (i * intervalSeconds * 1000)); + } } // Create Owntracks message @@ -199,7 +215,7 @@ public class GpxSender { // Wait before sending next point (except for the last one) if (i < trackPoints.size() - 1) { - Thread.sleep(intervalSeconds * 1000L); + Thread.sleep((long) (intervalSeconds * 1000)); } } } diff --git a/src/main/java/com/dedicatedcode/reitti/controller/settings/ManageDataController.java b/src/main/java/com/dedicatedcode/reitti/controller/settings/ManageDataController.java index d45ac520..b77aa55c 100644 --- a/src/main/java/com/dedicatedcode/reitti/controller/settings/ManageDataController.java +++ b/src/main/java/com/dedicatedcode/reitti/controller/settings/ManageDataController.java @@ -2,10 +2,7 @@ package com.dedicatedcode.reitti.controller.settings; import com.dedicatedcode.reitti.model.Role; import com.dedicatedcode.reitti.model.security.User; -import com.dedicatedcode.reitti.repository.ProcessedVisitJdbcService; -import com.dedicatedcode.reitti.repository.RawLocationPointJdbcService; -import com.dedicatedcode.reitti.repository.TripJdbcService; -import com.dedicatedcode.reitti.repository.VisitJdbcService; +import com.dedicatedcode.reitti.repository.*; import com.dedicatedcode.reitti.service.processing.ProcessingPipelineTrigger; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.MessageSource; @@ -25,13 +22,16 @@ public class ManageDataController { private final ProcessedVisitJdbcService processedVisitJdbcService; private final ProcessingPipelineTrigger processingPipelineTrigger; private final RawLocationPointJdbcService rawLocationPointJdbcService; + private final UserSettingsJdbcService userSettingsJdbcService; private final MessageSource messageSource; public ManageDataController(@Value("${reitti.data-management.enabled:false}") boolean dataManagementEnabled, VisitJdbcService visitJdbcService, TripJdbcService tripJdbcService, ProcessedVisitJdbcService processedVisitJdbcService, - ProcessingPipelineTrigger processingPipelineTrigger, RawLocationPointJdbcService rawLocationPointJdbcService, + ProcessingPipelineTrigger processingPipelineTrigger, + RawLocationPointJdbcService rawLocationPointJdbcService, + UserSettingsJdbcService userSettingsJdbcService, MessageSource messageSource) { this.dataManagementEnabled = dataManagementEnabled; this.visitJdbcService = visitJdbcService; @@ -39,6 +39,7 @@ public class ManageDataController { this.processedVisitJdbcService = processedVisitJdbcService; this.processingPipelineTrigger = processingPipelineTrigger; this.rawLocationPointJdbcService = rawLocationPointJdbcService; + this.userSettingsJdbcService = userSettingsJdbcService; this.messageSource = messageSource; } @@ -122,6 +123,7 @@ public class ManageDataController { } private void removeAllDataExceptPlaces(User user) { + this.userSettingsJdbcService.deleteNewestData(user); tripJdbcService.deleteAllForUser(user); processedVisitJdbcService.deleteAllForUser(user); visitJdbcService.deleteAllForUser(user); diff --git a/src/main/java/com/dedicatedcode/reitti/repository/UserSettingsJdbcService.java b/src/main/java/com/dedicatedcode/reitti/repository/UserSettingsJdbcService.java index 7fac22d4..9ed65fbe 100644 --- a/src/main/java/com/dedicatedcode/reitti/repository/UserSettingsJdbcService.java +++ b/src/main/java/com/dedicatedcode/reitti/repository/UserSettingsJdbcService.java @@ -111,4 +111,8 @@ public class UserSettingsJdbcService { public void deleteFor(User user) { this.jdbcTemplate.update("DELETE FROM user_settings WHERE user_id = ?", user.getId()); } + + public void deleteNewestData(User user) { + this.jdbcTemplate.update("UPDATE user_settings SET latest_data = NULL WHERE user_id = ?", user.getId()); + } } diff --git a/src/main/java/com/dedicatedcode/reitti/service/DefaultImportProcessor.java b/src/main/java/com/dedicatedcode/reitti/service/DefaultImportProcessor.java index f345492e..bb2cb1e3 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/DefaultImportProcessor.java +++ b/src/main/java/com/dedicatedcode/reitti/service/DefaultImportProcessor.java @@ -27,6 +27,7 @@ public class DefaultImportProcessor implements ImportProcessor { private final ProcessingPipelineTrigger processingPipelineTrigger; private final ScheduledExecutorService scheduler; private final ConcurrentHashMap> pendingTriggers; + private final ExecutorService importExecutors = Executors.newSingleThreadExecutor(); public DefaultImportProcessor( LocationDataIngestPipeline locationDataIngestPipeline, @@ -43,10 +44,13 @@ public class DefaultImportProcessor implements ImportProcessor { @Override public void processBatch(User user, List batch) { - logger.debug("Sending batch of {} locations for storing", batch.size()); - locationDataIngestPipeline.processLocationData(user.getUsername(), new ArrayList<>(batch)); - logger.debug("Sending batch of {} locations for processing", batch.size()); - scheduleProcessingTrigger(user.getUsername()); + logger.trace("Sending batch of [{}] locations for user [{}] into executor queue", batch.size(), user.getUsername()); + this.importExecutors.submit(() -> { + logger.debug("Sending batch of {} locations for storing", batch.size()); + locationDataIngestPipeline.processLocationData(user.getUsername(), new ArrayList<>(batch)); + logger.debug("Sending batch of {} locations for processing", batch.size()); + scheduleProcessingTrigger(user.getUsername()); + }); } @Override @@ -62,7 +66,6 @@ public class DefaultImportProcessor implements ImportProcessor { DefaultImportProcessor.logger.debug("Triggered processing for user: {}", username); TriggerProcessingEvent triggerEvent = new TriggerProcessingEvent(username, null, UUID.randomUUID().toString()); processingPipelineTrigger.handle(triggerEvent, false); - pendingTriggers.remove(username); } catch (Exception e) { DefaultImportProcessor.logger.error("Failed to trigger processing for user: {}", username, e); diff --git a/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleAndroidTimelineImporter.java b/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleAndroidTimelineImporter.java index bec97518..7f7dd4dc 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleAndroidTimelineImporter.java +++ b/src/main/java/com/dedicatedcode/reitti/service/importer/GoogleAndroidTimelineImporter.java @@ -64,7 +64,7 @@ public class GoogleAndroidTimelineImporter extends BaseGoogleTimelineImporter { if (semanticSegment.getTimelinePath() != null) { List timelinePath = semanticSegment.getTimelinePath(); - logger.info("Found timeline path from start [{}] to end [{}]. Will insert [{}] synthetic geo locations based on timeline path.", semanticSegment.getStartTime(), semanticSegment.getEndTime(), timelinePath.size()); + logger.info("Found timeline path from start [{}] to end [{}]. Will insert [{}] geo locations based on timeline path.", semanticSegment.getStartTime(), semanticSegment.getEndTime(), timelinePath.size()); for (TimelinePathPoint timelinePathPoint : timelinePath) { parseLatLng(timelinePathPoint.getPoint()).ifPresent(location -> { createAndScheduleLocationPoint(location, ZonedDateTime.parse(timelinePathPoint.getTime(), DateTimeFormatter.ISO_OFFSET_DATE_TIME).withNano(0), user, batch); diff --git a/src/main/java/com/dedicatedcode/reitti/service/processing/UnifiedLocationProcessingService.java b/src/main/java/com/dedicatedcode/reitti/service/processing/UnifiedLocationProcessingService.java index b35867d9..62482427 100644 --- a/src/main/java/com/dedicatedcode/reitti/service/processing/UnifiedLocationProcessingService.java +++ b/src/main/java/com/dedicatedcode/reitti/service/processing/UnifiedLocationProcessingService.java @@ -804,9 +804,9 @@ public class UnifiedLocationProcessingService { Point point = geometryFactory.createPoint(new Coordinate(longitude, latitude)); // Find places within the merge distance if (previewId == null) { - return significantPlaceJdbcService.findNearbyPlaces(user.getId(), point, GeoUtils.metersToDegreesAtPosition(mergeConfiguration.getMinDistanceBetweenVisits() / 2.0, latitude)); + return significantPlaceJdbcService.findNearbyPlaces(user.getId(), point, GeoUtils.metersToDegreesAtPosition(mergeConfiguration.getMinDistanceBetweenVisits(), latitude)); } else { - return previewSignificantPlaceJdbcService.findNearbyPlaces(user.getId(), point, GeoUtils.metersToDegreesAtPosition(mergeConfiguration.getMinDistanceBetweenVisits() / 2.0, latitude), previewId); + return previewSignificantPlaceJdbcService.findNearbyPlaces(user.getId(), point, GeoUtils.metersToDegreesAtPosition(mergeConfiguration.getMinDistanceBetweenVisits(), latitude), previewId); } }