mirror of
https://github.com/dedicatedcode/reitti.git
synced 2026-01-10 09:57:57 -05:00
Merge remote-tracking branch 'origin/main'
This commit is contained in:
Hosted Weblate
@@ -6,7 +6,7 @@ import org.springframework.stereotype.Component;
|
||||
@Component
|
||||
public class LocationDensityConfig {
|
||||
|
||||
@Value("${reitti.location.density.target-points-per-minute:2}")
|
||||
@Value("${reitti.location.density.target-points-per-minute:4}")
|
||||
private int targetPointsPerMinute;
|
||||
|
||||
public int getTargetPointsPerMinute() {
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.dedicatedcode.reitti.model;
|
||||
|
||||
import com.dedicatedcode.reitti.model.geo.RawLocationPoint;
|
||||
|
||||
public class ClusteredPoint {
|
||||
private final RawLocationPoint point;
|
||||
private final Integer clusterId;
|
||||
|
||||
public ClusteredPoint(RawLocationPoint point, Integer clusterId) {
|
||||
this.point = point;
|
||||
this.clusterId = clusterId;
|
||||
}
|
||||
|
||||
public RawLocationPoint getPoint() {
|
||||
return point;
|
||||
}
|
||||
|
||||
public Integer getClusterId() {
|
||||
return clusterId;
|
||||
}
|
||||
}
|
||||
@@ -14,18 +14,6 @@ public final class GeoUtils {
|
||||
private static final double EARTH_RADIUS = 6371000;
|
||||
|
||||
public static double distanceInMeters(double lat1, double lon1, double lat2, double lon2) {
|
||||
// For very small distances (< 1km), use faster approximation
|
||||
double latDiff = Math.abs(lat2 - lat1);
|
||||
double lonDiff = Math.abs(lon2 - lon1);
|
||||
|
||||
if (latDiff < 0.01 && lonDiff < 0.01) { // roughly < 1km
|
||||
double avgLat = Math.toRadians((lat1 + lat2) / 2);
|
||||
double latDistance = Math.toRadians(latDiff);
|
||||
double lonDistance = Math.toRadians(lonDiff) * Math.cos(avgLat);
|
||||
|
||||
return EARTH_RADIUS * Math.sqrt(latDistance * latDistance + lonDistance * lonDistance);
|
||||
}
|
||||
|
||||
// Use precise haversine formula for longer distances
|
||||
double latDistance = Math.toRadians(lat2 - lat1);
|
||||
double lonDistance = Math.toRadians(lon2 - lon1);
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.dedicatedcode.reitti.model.geo;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
|
||||
public class StayPoint {
|
||||
private final double latitude;
|
||||
private final double longitude;
|
||||
private final Instant arrivalTime;
|
||||
private final Instant departureTime;
|
||||
private final List<RawLocationPoint> points;
|
||||
|
||||
public StayPoint(double latitude, double longitude, Instant arrivalTime, Instant departureTime, List<RawLocationPoint> points) {
|
||||
this.latitude = latitude;
|
||||
this.longitude = longitude;
|
||||
this.arrivalTime = arrivalTime;
|
||||
this.departureTime = departureTime;
|
||||
this.points = points;
|
||||
}
|
||||
|
||||
public double getLatitude() {
|
||||
return latitude;
|
||||
}
|
||||
|
||||
public double getLongitude() {
|
||||
return longitude;
|
||||
}
|
||||
|
||||
public Instant getArrivalTime() {
|
||||
return arrivalTime;
|
||||
}
|
||||
|
||||
public Instant getDepartureTime() {
|
||||
return departureTime;
|
||||
}
|
||||
|
||||
public List<RawLocationPoint> getPoints() {
|
||||
return points;
|
||||
}
|
||||
|
||||
public long getDurationSeconds() {
|
||||
return Duration.between(arrivalTime, departureTime).getSeconds();
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,7 @@
|
||||
package com.dedicatedcode.reitti.repository;
|
||||
|
||||
import com.dedicatedcode.reitti.model.geo.GeoPoint;
|
||||
import com.dedicatedcode.reitti.model.ClusteredPoint;
|
||||
import com.dedicatedcode.reitti.model.geo.RawLocationPoint;
|
||||
import com.dedicatedcode.reitti.model.geo.Visit;
|
||||
import com.dedicatedcode.reitti.model.security.User;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
@@ -57,75 +56,33 @@ public class PreviewRawLocationPointJdbcService {
|
||||
"LIMIT ? OFFSET ?";
|
||||
return jdbcTemplate.query(sql, rawLocationPointRowMapper, user.getId(), previewId, limit, offset);
|
||||
}
|
||||
public List<Visit> findVisitsInTimerangeForUser(
|
||||
User user, String previewId, Instant startTime, Instant endTime, long minimumStayTime, double distanceInMeters) {
|
||||
String sql = """
|
||||
|
||||
WITH smoothed_data AS (
|
||||
-- Step 1: Smooth the track using a rolling centroid of the last 4 points
|
||||
SELECT
|
||||
*,
|
||||
ST_Centroid(
|
||||
ST_Collect(geom) OVER (
|
||||
ORDER BY "timestamp"
|
||||
ROWS BETWEEN 4 PRECEDING AND CURRENT ROW
|
||||
)
|
||||
) AS smoothed_geom
|
||||
FROM preview_raw_location_points
|
||||
WHERE user_id = ? AND preview_id = ? AND "timestamp" BETWEEN ? AND ?
|
||||
),
|
||||
lagged_data AS (
|
||||
-- Step 2: Add the lagged smoothed geometry and timestamp
|
||||
SELECT
|
||||
*,
|
||||
LAG(smoothed_geom, 4) OVER (ORDER BY "timestamp") AS prev_smoothed_geom,
|
||||
LAG("timestamp", 4) OVER (ORDER BY "timestamp") AS prev_ts
|
||||
FROM smoothed_data
|
||||
),
|
||||
island_flags AS (
|
||||
-- Step 3: Identify breaks (islands) based on smoothed movement
|
||||
SELECT
|
||||
*,
|
||||
CASE
|
||||
WHEN prev_smoothed_geom IS NULL THEN 1
|
||||
-- Check if current smoothed center is > 50m from 1 minute ago center
|
||||
WHEN ST_Distance(smoothed_geom::geography, prev_smoothed_geom::geography) > ? THEN 1
|
||||
-- Check if time gap is > 10 minutes (600 seconds)
|
||||
WHEN EXTRACT(EPOCH FROM ("timestamp" - prev_ts)) > ? THEN 1
|
||||
ELSE 0
|
||||
END AS is_new_cluster
|
||||
FROM lagged_data
|
||||
),
|
||||
clustered_points AS (
|
||||
-- Step 4: Assign a unique ID by summing the flags
|
||||
SELECT
|
||||
*,
|
||||
SUM(is_new_cluster) OVER (ORDER BY "timestamp") AS cluster_id
|
||||
FROM island_flags
|
||||
)
|
||||
-- Step 5: Final Output - Group into "Stay Events"
|
||||
SELECT
|
||||
cluster_id,
|
||||
MIN("timestamp") AS arrival,
|
||||
MAX("timestamp") AS departure,
|
||||
EXTRACT(EPOCH FROM (MAX("timestamp") - MIN("timestamp"))) AS duration,
|
||||
COUNT(*) AS point_count,
|
||||
ST_AsText(ST_Centroid(ST_Collect(geom))) AS cluster_center_geom
|
||||
FROM clustered_points
|
||||
GROUP BY cluster_id
|
||||
HAVING COUNT(*) >= 4 -- Must stay for at least 1 minute to be a cluster
|
||||
ORDER BY arrival;
|
||||
""";
|
||||
|
||||
public List<ClusteredPoint> findClusteredPointsInTimeRangeForUser(
|
||||
User user, String previewId, Instant startTime, Instant endTime, int minimumPoints, double distanceInMeters) {
|
||||
String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version , " +
|
||||
"ST_ClusterDBSCAN(rlp.geom, ?, ?) over () AS cluster_id " +
|
||||
"FROM preview_raw_location_points rlp " +
|
||||
"WHERE rlp.user_id = ? AND rlp.timestamp BETWEEN ? AND ? AND preview_id = ?";
|
||||
|
||||
return jdbcTemplate.query(sql, (rs, _) -> {
|
||||
GeoPoint geom = this.pointReaderWriter.read(rs.getString("cluster_center_geom"));
|
||||
return new Visit(geom.longitude(),
|
||||
geom.latitude(),
|
||||
rs.getTimestamp("arrival").toInstant(),
|
||||
rs.getTimestamp("departure").toInstant(),
|
||||
rs.getLong("duration"),
|
||||
false
|
||||
);
|
||||
}, user.getId(), previewId, Timestamp.from(startTime), Timestamp.from(endTime), distanceInMeters, minimumStayTime);
|
||||
|
||||
RawLocationPoint point = new RawLocationPoint(
|
||||
rs.getLong("id"),
|
||||
rs.getTimestamp("timestamp").toInstant(),
|
||||
this.pointReaderWriter.read(rs.getString("geom")),
|
||||
rs.getDouble("accuracy_meters"),
|
||||
rs.getObject("elevation_meters", Double.class),
|
||||
rs.getBoolean("processed"),
|
||||
rs.getBoolean("synthetic"),
|
||||
rs.getBoolean("ignored"),
|
||||
rs.getLong("version")
|
||||
);
|
||||
|
||||
Integer clusterId = rs.getObject("cluster_id", Integer.class);
|
||||
|
||||
return new ClusteredPoint(point, clusterId);
|
||||
}, distanceInMeters, minimumPoints, user.getId(),
|
||||
Timestamp.from(startTime), Timestamp.from(endTime), previewId);
|
||||
}
|
||||
|
||||
public void bulkUpdateProcessedStatus(List<RawLocationPoint> points) {
|
||||
|
||||
@@ -63,6 +63,20 @@ public class ProcessedVisitJdbcService {
|
||||
Timestamp.from(endTime), Timestamp.from(startTime));
|
||||
}
|
||||
|
||||
public Optional<ProcessedVisit> findFirstProcessedVisitBefore(User user, Instant time) {
|
||||
String sql = "SELECT pv.* " +
|
||||
"FROM processed_visits pv " +
|
||||
"WHERE pv.user_id = ? AND pv.end_time < ? ORDER BY pv.end_time DESC LIMIT 1";
|
||||
return jdbcTemplate.query(sql, PROCESSED_VISIT_ROW_MAPPER, user.getId(), Timestamp.from(time)).stream().findFirst();
|
||||
}
|
||||
|
||||
public Optional<ProcessedVisit> findFirstProcessedVisitAfter(User user, Instant time) {
|
||||
String sql = "SELECT pv.* " +
|
||||
"FROM processed_visits pv " +
|
||||
"WHERE pv.user_id = ? AND pv.start_time > ? ORDER BY pv.start_time LIMIT 1";
|
||||
return jdbcTemplate.query(sql, PROCESSED_VISIT_ROW_MAPPER, user.getId(), Timestamp.from(time)).stream().findFirst();
|
||||
|
||||
}
|
||||
public Optional<ProcessedVisit> findByUserAndId(User user, long id) {
|
||||
String sql = "SELECT pv.* " +
|
||||
"FROM processed_visits pv " +
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
package com.dedicatedcode.reitti.repository;
|
||||
|
||||
import com.dedicatedcode.reitti.dto.LocationPoint;
|
||||
import com.dedicatedcode.reitti.model.geo.GeoPoint;
|
||||
import com.dedicatedcode.reitti.model.ClusteredPoint;
|
||||
import com.dedicatedcode.reitti.model.geo.RawLocationPoint;
|
||||
import com.dedicatedcode.reitti.model.geo.Visit;
|
||||
import com.dedicatedcode.reitti.model.security.User;
|
||||
import org.locationtech.jts.geom.Coordinate;
|
||||
import org.locationtech.jts.geom.GeometryFactory;
|
||||
@@ -15,10 +14,7 @@ import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
@@ -36,7 +32,7 @@ public class RawLocationPointJdbcService {
|
||||
|
||||
public RawLocationPointJdbcService(JdbcTemplate jdbcTemplate, PointReaderWriter pointReaderWriter, GeometryFactory geometryFactory) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.rawLocationPointRowMapper = (rs, _) -> new RawLocationPoint(
|
||||
this.rawLocationPointRowMapper = (rs, rowNum) -> new RawLocationPoint(
|
||||
rs.getLong("id"),
|
||||
rs.getTimestamp("timestamp").toInstant(),
|
||||
pointReaderWriter.read(rs.getString("geom")),
|
||||
@@ -169,7 +165,7 @@ public class RawLocationPointJdbcService {
|
||||
"FROM raw_location_points rlp " +
|
||||
"WHERE rlp.id = ?";
|
||||
List<RawLocationPoint> results = jdbcTemplate.query(sql, rawLocationPointRowMapper, id);
|
||||
return results.isEmpty() ? Optional.empty() : Optional.of(results.getFirst());
|
||||
return results.isEmpty() ? Optional.empty() : Optional.of(results.get(0));
|
||||
}
|
||||
|
||||
public Optional<RawLocationPoint> findLatest(User user, Instant since) {
|
||||
@@ -178,7 +174,7 @@ public class RawLocationPointJdbcService {
|
||||
"WHERE rlp.user_id = ? AND rlp.timestamp >= ? " +
|
||||
"ORDER BY rlp.timestamp LIMIT 1";
|
||||
List<RawLocationPoint> results = jdbcTemplate.query(sql, rawLocationPointRowMapper, user.getId(), Timestamp.from(since));
|
||||
return results.isEmpty() ? Optional.empty() : Optional.of(results.getFirst());
|
||||
return results.isEmpty() ? Optional.empty() : Optional.of(results.get(0));
|
||||
}
|
||||
|
||||
public Optional<RawLocationPoint> findLatest(User user) {
|
||||
@@ -187,86 +183,41 @@ public class RawLocationPointJdbcService {
|
||||
"WHERE rlp.user_id = ? " +
|
||||
"ORDER BY rlp.timestamp DESC LIMIT 1";
|
||||
List<RawLocationPoint> results = jdbcTemplate.query(sql, rawLocationPointRowMapper, user.getId());
|
||||
return results.isEmpty() ? Optional.empty() : Optional.of(results.getFirst());
|
||||
return results.isEmpty() ? Optional.empty() : Optional.of(results.get(0));
|
||||
}
|
||||
|
||||
public List<Visit> findVisitsInTimerangeForUser(
|
||||
User user, Instant startTime, Instant endTime, long minimumStayTime, double distanceInMeters) {
|
||||
String sql = """
|
||||
|
||||
WITH smoothed_data AS (
|
||||
-- Step 1: Smooth the track using a rolling centroid of the last 4 points
|
||||
SELECT\s
|
||||
*,
|
||||
ST_Centroid(
|
||||
ST_Collect(geom) OVER (
|
||||
ORDER BY "timestamp"
|
||||
ROWS BETWEEN 4 PRECEDING AND CURRENT ROW
|
||||
)
|
||||
) AS smoothed_geom
|
||||
FROM raw_location_points
|
||||
WHERE user_id = ? AND "timestamp" BETWEEN ? AND ?
|
||||
),
|
||||
lagged_data AS (
|
||||
-- Step 2: Add the lagged smoothed geometry and timestamp
|
||||
SELECT
|
||||
*,
|
||||
LAG(smoothed_geom, 4) OVER (ORDER BY "timestamp") AS prev_smoothed_geom,
|
||||
LAG("timestamp", 4) OVER (ORDER BY "timestamp") AS prev_ts
|
||||
FROM smoothed_data
|
||||
),
|
||||
island_flags AS (
|
||||
-- Step 3: Identify breaks (islands) based on smoothed movement
|
||||
SELECT
|
||||
*,
|
||||
CASE
|
||||
WHEN prev_smoothed_geom IS NULL THEN 1
|
||||
-- Check if current smoothed center is > 50m from 1 minute ago center
|
||||
WHEN ST_Distance(smoothed_geom::geography, prev_smoothed_geom::geography) > ? THEN 1
|
||||
-- Check if time gap is > 10 minutes (600 seconds)
|
||||
WHEN EXTRACT(EPOCH FROM ("timestamp" - prev_ts)) > ? THEN 1
|
||||
ELSE 0
|
||||
END AS is_new_cluster
|
||||
FROM lagged_data
|
||||
),
|
||||
clustered_points AS (
|
||||
-- Step 4: Assign a unique ID by summing the flags
|
||||
SELECT
|
||||
*,
|
||||
SUM(is_new_cluster) OVER (ORDER BY "timestamp") AS cluster_id
|
||||
FROM island_flags
|
||||
)
|
||||
-- Step 5: Final Output - Group into "Stay Events"
|
||||
SELECT
|
||||
cluster_id,
|
||||
MIN("timestamp") AS arrival,
|
||||
MAX("timestamp") AS departure,
|
||||
EXTRACT(EPOCH FROM (MAX("timestamp") - MIN("timestamp"))) AS duration,
|
||||
COUNT(*) AS point_count,
|
||||
ST_AsText(ST_Centroid(ST_Collect(geom))) AS cluster_center_geom
|
||||
FROM clustered_points
|
||||
GROUP BY cluster_id
|
||||
HAVING COUNT(*) >= 4 -- Must stay for at least 1 minute to be a cluster
|
||||
ORDER BY arrival;
|
||||
""";
|
||||
return jdbcTemplate.query(sql, (rs, _) -> {
|
||||
GeoPoint geom = this.pointReaderWriter.read(rs.getString("cluster_center_geom"));
|
||||
return new Visit(geom.longitude(),
|
||||
geom.latitude(),
|
||||
rs.getTimestamp("arrival").toInstant(),
|
||||
rs.getTimestamp("departure").toInstant(),
|
||||
rs.getLong("duration"),
|
||||
false
|
||||
);
|
||||
}, user.getId(), Timestamp.from(startTime), Timestamp.from(endTime), distanceInMeters, minimumStayTime);
|
||||
public List<ClusteredPoint> findClusteredPointsInTimeRangeForUser(
|
||||
User user, Instant startTime, Instant endTime, int minimumPoints, double distanceInMeters) {
|
||||
String sql = "SELECT rlp.id, rlp.accuracy_meters, rlp.elevation_meters, rlp.timestamp, rlp.user_id, ST_AsText(rlp.geom) as geom, rlp.processed, rlp.synthetic, rlp.ignored, rlp.version , " +
|
||||
"ST_ClusterDBSCAN(rlp.geom, ?, ?) over () AS cluster_id " +
|
||||
"FROM raw_location_points rlp " +
|
||||
"WHERE rlp.user_id = ? AND rlp.timestamp BETWEEN ? AND ?";
|
||||
|
||||
return jdbcTemplate.query(sql, (rs, rowNum) -> {
|
||||
|
||||
RawLocationPoint point = new RawLocationPoint(
|
||||
rs.getLong("id"),
|
||||
rs.getTimestamp("timestamp").toInstant(),
|
||||
this.pointReaderWriter.read(rs.getString("geom")),
|
||||
rs.getDouble("accuracy_meters"),
|
||||
rs.getObject("elevation_meters", Double.class),
|
||||
rs.getBoolean("processed"),
|
||||
rs.getBoolean("synthetic"),
|
||||
rs.getBoolean("ignored"),
|
||||
rs.getLong("version")
|
||||
);
|
||||
|
||||
Integer clusterId = rs.getObject("cluster_id", Integer.class);
|
||||
|
||||
return new ClusteredPoint(point, clusterId);
|
||||
}, distanceInMeters, minimumPoints, user.getId(),
|
||||
Timestamp.from(startTime), Timestamp.from(endTime));
|
||||
}
|
||||
|
||||
@SuppressWarnings("DataFlowIssue")
|
||||
public long count() {
|
||||
return jdbcTemplate.queryForObject("SELECT COUNT(*) FROM raw_location_points", Long.class);
|
||||
}
|
||||
|
||||
@SuppressWarnings("DataFlowIssue")
|
||||
public List<RawLocationPoint> findPointsInBoxWithNeighbors(
|
||||
User user,
|
||||
Instant startTime,
|
||||
@@ -441,7 +392,7 @@ public class RawLocationPointJdbcService {
|
||||
String sql = """
|
||||
WITH sampled_points AS (
|
||||
SELECT DISTINCT ON (
|
||||
date_trunc('hour', timestamp) +
|
||||
date_trunc('hour', timestamp) +
|
||||
(EXTRACT(minute FROM timestamp)::int / %d) * interval '%d minutes'
|
||||
)
|
||||
id,
|
||||
@@ -458,7 +409,7 @@ public class RawLocationPointJdbcService {
|
||||
AND timestamp BETWEEN ? AND ?
|
||||
AND ignored = false
|
||||
ORDER BY
|
||||
date_trunc('hour', timestamp) +
|
||||
date_trunc('hour', timestamp) +
|
||||
(EXTRACT(minute FROM timestamp)::int / %d) * interval '%d minutes',
|
||||
timestamp
|
||||
)
|
||||
@@ -482,7 +433,6 @@ public class RawLocationPointJdbcService {
|
||||
Timestamp.from(startTime), Timestamp.from(endTime));
|
||||
}
|
||||
|
||||
@SuppressWarnings("DataFlowIssue")
|
||||
public long countByUser(User user) {
|
||||
return jdbcTemplate.queryForObject("SELECT COUNT(*) FROM raw_location_points WHERE user_id = ?", Long.class, user.getId());
|
||||
}
|
||||
|
||||
@@ -47,8 +47,17 @@ public class SyntheticLocationPointGenerator {
|
||||
// Calculate interpolation ratio (0.0 to 1.0)
|
||||
long totalDuration = endTime.getEpochSecond() - startTime.getEpochSecond();
|
||||
long currentDuration = currentTime.getEpochSecond() - startTime.getEpochSecond();
|
||||
double ratio = (double) currentDuration / totalDuration;
|
||||
|
||||
double timeRatio = (double) currentDuration / totalDuration;
|
||||
|
||||
// Calculate speed and adjust the ratio based on it
|
||||
double distance = GeoUtils.distanceInMeters(startPoint, endPoint);
|
||||
double speed = distance / totalDuration; // meters per second
|
||||
|
||||
// Apply speed-based transformation: slower speeds shift points towards start
|
||||
// Using a power function where speed < 1 m/s creates stronger clustering at start
|
||||
double speedFactor = Math.min(speed / 5.0, 1.0); // normalize to 5 m/s as reference
|
||||
double ratio = Math.pow(timeRatio, 1.0 / (speedFactor + 0.5));
|
||||
|
||||
// Interpolate coordinates
|
||||
GeoPoint interpolatedCoords = interpolateCoordinates(
|
||||
startPoint.getGeom(),
|
||||
|
||||
@@ -3,6 +3,7 @@ package com.dedicatedcode.reitti.service.processing;
|
||||
import com.dedicatedcode.reitti.config.RabbitMQConfig;
|
||||
import com.dedicatedcode.reitti.event.LocationProcessEvent;
|
||||
import com.dedicatedcode.reitti.event.SignificantPlaceCreatedEvent;
|
||||
import com.dedicatedcode.reitti.model.ClusteredPoint;
|
||||
import com.dedicatedcode.reitti.model.PlaceInformationOverride;
|
||||
import com.dedicatedcode.reitti.model.geo.*;
|
||||
import com.dedicatedcode.reitti.model.processing.DetectionParameter;
|
||||
@@ -23,10 +24,8 @@ import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Unified service that processes the entire GPS pipeline atomically per user.
|
||||
@@ -102,8 +101,7 @@ public class UnifiedLocationProcessingService {
|
||||
String username = event.getUsername();
|
||||
String previewId = event.getPreviewId();
|
||||
|
||||
logger.info("Processing location data for user [{}], mode: {}",
|
||||
username, previewId == null ? "LIVE" : "PREVIEW");
|
||||
logger.info("Processing location data for user [{}], mode: {}", username, previewId == null ? "LIVE" : "PREVIEW");
|
||||
|
||||
User user = userJdbcService.findByUsername(username)
|
||||
.orElseThrow(() -> new IllegalStateException("User not found: " + username));
|
||||
@@ -147,7 +145,7 @@ public class UnifiedLocationProcessingService {
|
||||
}
|
||||
|
||||
long duration = System.currentTimeMillis() - startTime;
|
||||
|
||||
|
||||
if (logger.isTraceEnabled()) {
|
||||
// Tabular output for trace level logging
|
||||
StringBuilder traceOutput = new StringBuilder();
|
||||
@@ -155,13 +153,13 @@ public class UnifiedLocationProcessingService {
|
||||
traceOutput.append("Event Period: ").append(event.getEarliest()).append(" → ").append(event.getLatest()).append("\n");
|
||||
traceOutput.append("Search Period: ").append(detectionResult.searchStart).append(" → ").append(detectionResult.searchEnd).append("\n");
|
||||
traceOutput.append("Duration: ").append(duration).append("ms\n\n");
|
||||
|
||||
|
||||
// Input Visits Table
|
||||
traceOutput.append("INPUT VISITS (").append(mergingResult.inputVisits.size()).append(") - took [").append(detectionResult.durationMillis).append("]ms:\n");
|
||||
traceOutput.append("INPUT VISITS (").append(detectionResult.visits.size()).append(") - took [").append(detectionResult.durationInMillis).append("]ms:\n");
|
||||
traceOutput.append("┌─────────────────────┬─────────────────────┬───────────┬─────────────┬─────────────┬──────────────────────────────────────────────────────────────────────┐\n");
|
||||
traceOutput.append("│ Start Time │ End Time │ Duration │ Latitude │ Longitude │ Google Maps Link │\n");
|
||||
traceOutput.append("├─────────────────────┼─────────────────────┼───────────┼─────────────┼─────────────┼──────────────────────────────────────────────────────────────────────┤\n");
|
||||
for (Visit visit : mergingResult.inputVisits) {
|
||||
for (Visit visit : detectionResult.visits) {
|
||||
String googleMapsLink = "https://www.google.com/maps/search/?api=1&query=" + visit.getLatitude() + "," + visit.getLongitude();
|
||||
traceOutput.append(String.format("│ %-19s │ %-19s │ %8ds │ %11.6f │ %11.6f │ %-68s │\n",
|
||||
visit.getStartTime().toString().substring(0, 19),
|
||||
@@ -172,9 +170,9 @@ public class UnifiedLocationProcessingService {
|
||||
googleMapsLink));
|
||||
}
|
||||
traceOutput.append("└─────────────────────┴─────────────────────┴───────────┴─────────────┴─────────────┴──────────────────────────────────────────────────────────────────────┘\n\n");
|
||||
|
||||
|
||||
// Processed Visits Table
|
||||
traceOutput.append("PROCESSED VISITS (").append(mergingResult.processedVisits.size()).append(") - took [").append(mergingResult.durationMillis).append("]ms:\n");
|
||||
traceOutput.append("PROCESSED VISITS (").append(mergingResult.processedVisits.size()).append(") - took [").append(mergingResult.durationInMillis).append("]ms:\n");
|
||||
traceOutput.append("┌─────────────────────┬─────────────────────┬───────────┬─────────────┬─────────────┬──────────────────────┐\n");
|
||||
traceOutput.append("│ Start Time │ End Time │ Duration │ Latitude │ Longitude │ Place Name │\n");
|
||||
traceOutput.append("├─────────────────────┼─────────────────────┼───────────┼─────────────┼─────────────┼──────────────────────┤\n");
|
||||
@@ -190,9 +188,9 @@ public class UnifiedLocationProcessingService {
|
||||
placeName));
|
||||
}
|
||||
traceOutput.append("└─────────────────────┴─────────────────────┴───────────┴─────────────┴─────────────┴──────────────────────┘\n\n");
|
||||
|
||||
|
||||
// Trips Table
|
||||
traceOutput.append("TRIPS (").append(tripResult.trips.size()).append(") - took [").append(tripResult.durationMillis).append("]ms:\n");
|
||||
traceOutput.append("TRIPS (").append(tripResult.trips.size()).append(") - took [").append(tripResult.durationInMillis).append("]ms:\n");
|
||||
traceOutput.append("┌─────────────────────┬─────────────────────┬───────────┬───────────┬───────────┬─────────────────┐\n");
|
||||
traceOutput.append("│ Start Time │ End Time │ Duration │ Distance │ Traveled │ Transport Mode │\n");
|
||||
traceOutput.append("├─────────────────────┼─────────────────────┼───────────┼───────────┼───────────┼─────────────────┤\n");
|
||||
@@ -206,10 +204,10 @@ public class UnifiedLocationProcessingService {
|
||||
trip.getTransportModeInferred().toString()));
|
||||
}
|
||||
traceOutput.append("└─────────────────────┴─────────────────────┴───────────┴───────────┴───────────┴─────────────────┘\n");
|
||||
|
||||
|
||||
logger.trace(traceOutput.toString());
|
||||
}
|
||||
|
||||
|
||||
logger.info("Completed processing for user [{}] in {}ms: {} visits → {} processed visits → {} trips",
|
||||
username, duration, detectionResult.visits.size(),
|
||||
mergingResult.processedVisits.size(), tripResult.trips.size());
|
||||
@@ -245,7 +243,7 @@ public class UnifiedLocationProcessingService {
|
||||
.findByUserAndStartTimeBeforeEqualAndEndTimeAfterEqual(user, previewId, windowEnd, windowStart);
|
||||
}
|
||||
|
||||
// Expand the window based on found processed visits
|
||||
// Expand window based on deleted processed visits
|
||||
if (!existingProcessedVisits.isEmpty()) {
|
||||
if (existingProcessedVisits.getFirst().getStartTime().isBefore(windowStart)) {
|
||||
windowStart = existingProcessedVisits.getFirst().getStartTime();
|
||||
@@ -254,15 +252,48 @@ public class UnifiedLocationProcessingService {
|
||||
windowEnd = existingProcessedVisits.getLast().getEndTime();
|
||||
}
|
||||
}
|
||||
List<Visit> visits;
|
||||
|
||||
// Get clustered points
|
||||
double baseLatitude = existingProcessedVisits.isEmpty() ? 50 : existingProcessedVisits.getFirst().getPlace().getLatitudeCentroid();
|
||||
double metersAsDegrees = GeoUtils.metersToDegreesAtPosition((double) currentConfiguration.getVisitMerging().getMinDistanceBetweenVisits() / 4, baseLatitude);
|
||||
|
||||
List<ClusteredPoint> clusteredPoints;
|
||||
int minimumAdjacentPoints = Math.max(4, Math.toIntExact(detectionParams.getMinimumStayTimeInSeconds() / 60));
|
||||
if (previewId == null) {
|
||||
visits = rawLocationPointJdbcService.findVisitsInTimerangeForUser(
|
||||
user, windowStart, windowEnd, detectionParams.getMinimumStayTimeInSeconds(), currentConfiguration.getVisitMerging().getMinDistanceBetweenVisits());
|
||||
clusteredPoints = rawLocationPointJdbcService.findClusteredPointsInTimeRangeForUser(
|
||||
user, windowStart, windowEnd, minimumAdjacentPoints, metersAsDegrees);
|
||||
} else {
|
||||
visits = previewRawLocationPointJdbcService.findVisitsInTimerangeForUser(
|
||||
user, previewId, windowStart, windowEnd, detectionParams.getMinimumStayTimeInSeconds(), currentConfiguration.getVisitMerging().getMinDistanceBetweenVisits());
|
||||
clusteredPoints = previewRawLocationPointJdbcService.findClusteredPointsInTimeRangeForUser(
|
||||
user, previewId, windowStart, windowEnd, minimumAdjacentPoints, metersAsDegrees);
|
||||
}
|
||||
visits.sort(Comparator.comparing(Visit::getStartTime));
|
||||
logger.debug("Searching for clustered points in range [{}, {}], minimum adjacent points: {} ", windowStart, windowEnd, minimumAdjacentPoints);
|
||||
|
||||
// Cluster by location and time
|
||||
Map<Integer, List<RawLocationPoint>> clusteredByLocation = new TreeMap<>();
|
||||
for (ClusteredPoint cp : clusteredPoints.stream().filter(clusteredPoint -> !clusteredPoint.getPoint().isIgnored()).toList()) {
|
||||
if (cp.getClusterId() != null) {
|
||||
clusteredByLocation
|
||||
.computeIfAbsent(cp.getClusterId(), _ -> new ArrayList<>())
|
||||
.add(cp.getPoint());
|
||||
}
|
||||
}
|
||||
|
||||
// Detect stay points
|
||||
List<StayPoint> stayPoints = detectStayPointsFromTrajectory(clusteredByLocation, detectionParams);
|
||||
|
||||
// Create visits
|
||||
List<Visit> visits = stayPoints.stream()
|
||||
.map(sp -> new Visit(
|
||||
sp.getLongitude(),
|
||||
sp.getLatitude(),
|
||||
sp.getArrivalTime(),
|
||||
sp.getDepartureTime(),
|
||||
sp.getDurationSeconds(),
|
||||
false
|
||||
))
|
||||
.sorted(Comparator.comparing(Visit::getStartTime))
|
||||
.toList();
|
||||
|
||||
return new VisitDetectionResult(visits, windowStart, windowEnd, System.currentTimeMillis() - start);
|
||||
}
|
||||
|
||||
@@ -315,7 +346,6 @@ public class UnifiedLocationProcessingService {
|
||||
return new VisitMergingResult(List.of(), List.of(), searchStart, searchEnd, System.currentTimeMillis() - start);
|
||||
}
|
||||
|
||||
allVisits = allVisits.stream().sorted(Comparator.comparing(Visit::getStartTime)).toList();
|
||||
// Merge visits chronologically
|
||||
List<ProcessedVisit> processedVisits = mergeVisitsChronologically(
|
||||
user, previewId, traceId, allVisits, mergeConfig);
|
||||
@@ -363,6 +393,19 @@ public class UnifiedLocationProcessingService {
|
||||
}
|
||||
}
|
||||
|
||||
if (previewId == null) {
|
||||
//recreate the trip between this run's first visit and the processed visit before. We deleted that when we cleared the processed visits in the search range. But only if it is max 24h apart
|
||||
Optional<ProcessedVisit> firstProcessedVisitBefore = this.processedVisitJdbcService.findFirstProcessedVisitBefore(user, searchStart);
|
||||
if (firstProcessedVisitBefore.isPresent() && Duration.between(firstProcessedVisitBefore.get().getEndTime(), processedVisits.getFirst().getStartTime()).toHours() <= 24) {
|
||||
trips.add(createTripBetweenVisits(user, null, firstProcessedVisitBefore.get(), processedVisits.getFirst()));
|
||||
}
|
||||
|
||||
Optional<ProcessedVisit> processedVisitAfter = this.processedVisitJdbcService.findFirstProcessedVisitAfter(user, searchEnd);
|
||||
if (processedVisitAfter.isPresent() && Duration.between(processedVisits.getLast().getEndTime(), processedVisitAfter.get().getStartTime()).toHours() <= 24) {
|
||||
trips.add(createTripBetweenVisits(user, null, processedVisits.getLast(), processedVisitAfter.get()));
|
||||
}
|
||||
}
|
||||
trips.sort(Comparator.comparing(Trip::getStartTime));
|
||||
// Save trips
|
||||
if (previewId == null) {
|
||||
trips = tripJdbcService.bulkInsert(user, trips);
|
||||
@@ -373,6 +416,52 @@ public class UnifiedLocationProcessingService {
|
||||
return new TripDetectionResult(trips, System.currentTimeMillis() - start);
|
||||
}
|
||||
|
||||
private List<StayPoint> detectStayPointsFromTrajectory(
|
||||
Map<Integer, List<RawLocationPoint>> points,
|
||||
DetectionParameter.VisitDetection visitDetectionParameters) {
|
||||
logger.debug("Starting cluster-based stay point detection with {} different spatial clusters.", points.size());
|
||||
|
||||
List<List<RawLocationPoint>> clusters = new ArrayList<>();
|
||||
|
||||
//split them up when time is x seconds between
|
||||
for (List<RawLocationPoint> clusteredByLocation : points.values()) {
|
||||
logger.debug("Start splitting up geospatial cluster with [{}] elements based on minimum time [{}]s between points", clusteredByLocation.size(), visitDetectionParameters.getMaxMergeTimeBetweenSameStayPoints());
|
||||
//first sort them by timestamp
|
||||
clusteredByLocation.sort(Comparator.comparing(RawLocationPoint::getTimestamp));
|
||||
|
||||
List<RawLocationPoint> currentTimedCluster = new ArrayList<>();
|
||||
clusters.add(currentTimedCluster);
|
||||
currentTimedCluster.add(clusteredByLocation.getFirst());
|
||||
|
||||
Instant currentTime = clusteredByLocation.getFirst().getTimestamp();
|
||||
|
||||
for (int i = 1; i < clusteredByLocation.size(); i++) {
|
||||
RawLocationPoint next = clusteredByLocation.get(i);
|
||||
if (Duration.between(currentTime, next.getTimestamp()).getSeconds() < visitDetectionParameters.getMaxMergeTimeBetweenSameStayPoints()) {
|
||||
currentTimedCluster.add(next);
|
||||
} else {
|
||||
currentTimedCluster = new ArrayList<>();
|
||||
currentTimedCluster.add(next);
|
||||
clusters.add(currentTimedCluster);
|
||||
}
|
||||
currentTime = next.getTimestamp();
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug("Detected {} stay points after splitting them up.", clusters.size());
|
||||
//filter them by duration
|
||||
List<List<RawLocationPoint>> filteredByMinimumDuration = clusters.stream()
|
||||
.filter(c -> Duration.between(c.getFirst().getTimestamp(), c.getLast().getTimestamp()).toSeconds() > visitDetectionParameters.getMinimumStayTimeInSeconds())
|
||||
.toList();
|
||||
|
||||
logger.debug("Found {} valid clusters after duration filtering with minimum stay time [{}]s", filteredByMinimumDuration.size(), visitDetectionParameters.getMinimumStayTimeInSeconds());
|
||||
|
||||
// Step 3: Convert valid clusters to stay points
|
||||
return filteredByMinimumDuration.stream()
|
||||
.map(this::createStayPoint)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private List<ProcessedVisit> mergeVisitsChronologically(
|
||||
User user, String previewId, String traceId, List<Visit> visits,
|
||||
DetectionParameter.VisitMerging mergeConfiguration) {
|
||||
@@ -394,13 +483,14 @@ public class UnifiedLocationProcessingService {
|
||||
Visit nextVisit = visits.get(i);
|
||||
|
||||
if (nextVisit.getStartTime().isBefore(currentEndTime)) {
|
||||
logger.debug("Skipping visit [{}] because it starts before the end time of the previous one [{}]", nextVisit, currentEndTime);
|
||||
logger.error("Skipping visit [{}] because it starts before the end time of the previous one [{}]", nextVisit, currentEndTime);
|
||||
continue;
|
||||
}
|
||||
SignificantPlace nextPlace = findOrCreateSignificantPlace(user, previewId, nextVisit.getLatitude(), nextVisit.getLongitude(), mergeConfiguration, traceId);
|
||||
|
||||
boolean samePlace = nextPlace.getId().equals(currentPlace.getId());
|
||||
boolean withinTimeThreshold = Duration.between(currentEndTime, nextVisit.getStartTime()).getSeconds() <= mergeConfiguration.getMaxMergeTimeBetweenSameVisits();
|
||||
|
||||
boolean shouldMergeWithNextVisit = samePlace && withinTimeThreshold;
|
||||
|
||||
if (samePlace && !withinTimeThreshold) {
|
||||
@@ -413,7 +503,7 @@ public class UnifiedLocationProcessingService {
|
||||
if (pointsBetweenVisits.size() > 2) {
|
||||
double travelledDistanceInMeters = GeoUtils.calculateTripDistance(pointsBetweenVisits);
|
||||
shouldMergeWithNextVisit = travelledDistanceInMeters <= mergeConfiguration.getMinDistanceBetweenVisits();
|
||||
} else {
|
||||
} else {
|
||||
logger.debug("There are no points tracked between {} and {}. Will merge consecutive visits because they are on the same place", currentEndTime, nextVisit.getStartTime());
|
||||
shouldMergeWithNextVisit = true;
|
||||
}
|
||||
@@ -470,6 +560,166 @@ public class UnifiedLocationProcessingService {
|
||||
return new ProcessedVisit(place, startTime, endTime, endTime.getEpochSecond() - startTime.getEpochSecond());
|
||||
}
|
||||
|
||||
private StayPoint createStayPoint(List<RawLocationPoint> clusterPoints) {
|
||||
GeoPoint result = weightedCenter(clusterPoints);
|
||||
|
||||
// Get the time range
|
||||
Instant arrivalTime = clusterPoints.getFirst().getTimestamp();
|
||||
Instant departureTime = clusterPoints.getLast().getTimestamp();
|
||||
|
||||
logger.trace("Creating stay point at [{}] with arrival time [{}] and departure time [{}]", result, arrivalTime, departureTime);
|
||||
return new StayPoint(result.latitude(), result.longitude(), arrivalTime, departureTime, clusterPoints);
|
||||
}
|
||||
|
||||
private GeoPoint weightedCenter(List<RawLocationPoint> clusterPoints) {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
|
||||
GeoPoint result;
|
||||
// For small clusters, use the original algorithm
|
||||
if (clusterPoints.size() <= 100) {
|
||||
result = weightedCenterSimple(clusterPoints);
|
||||
} else {
|
||||
// For large clusters, use spatial partitioning for better performance
|
||||
result = weightedCenterOptimized(clusterPoints);
|
||||
}
|
||||
logger.trace("Weighted center calculation took {}ms for [{}] number of points", System.currentTimeMillis() - start, clusterPoints.size());
|
||||
return result;
|
||||
}
|
||||
|
||||
private GeoPoint weightedCenterSimple(List<RawLocationPoint> clusterPoints) {
|
||||
RawLocationPoint bestPoint = null;
|
||||
double maxDensityScore = 0;
|
||||
|
||||
// For each point, calculate a density score based on nearby points and accuracy
|
||||
for (RawLocationPoint candidate : clusterPoints) {
|
||||
double densityScore = 0;
|
||||
|
||||
for (RawLocationPoint neighbor : clusterPoints) {
|
||||
if (candidate == neighbor) continue;
|
||||
|
||||
double distance = GeoUtils.distanceInMeters(candidate, neighbor);
|
||||
double accuracy = candidate.getAccuracyMeters() != null && candidate.getAccuracyMeters() > 0
|
||||
? candidate.getAccuracyMeters()
|
||||
: 50.0; // default accuracy if null
|
||||
|
||||
// Points within accuracy radius contribute to density
|
||||
// Closer points and better accuracy contribute more
|
||||
if (distance <= accuracy * 2) {
|
||||
double proximityWeight = Math.max(0, 1.0 - (distance / (accuracy * 2)));
|
||||
double accuracyWeight = 1.0 / accuracy;
|
||||
densityScore += proximityWeight * accuracyWeight;
|
||||
}
|
||||
}
|
||||
|
||||
// Add self-contribution based on accuracy
|
||||
densityScore += 1.0 / (candidate.getAccuracyMeters() != null && candidate.getAccuracyMeters() > 0
|
||||
? candidate.getAccuracyMeters()
|
||||
: 50.0);
|
||||
|
||||
if (densityScore > maxDensityScore) {
|
||||
maxDensityScore = densityScore;
|
||||
bestPoint = candidate;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to first point if no best point found
|
||||
if (bestPoint == null) {
|
||||
bestPoint = clusterPoints.getFirst();
|
||||
}
|
||||
|
||||
return new GeoPoint(bestPoint.getLatitude(), bestPoint.getLongitude());
|
||||
}
|
||||
|
||||
private GeoPoint weightedCenterOptimized(List<RawLocationPoint> clusterPoints) {
|
||||
// Sample a subset of points for density calculation to improve performance
|
||||
// Use every nth point or random sampling for very large clusters
|
||||
int sampleSize = Math.min(200, clusterPoints.size());
|
||||
List<RawLocationPoint> samplePoints = new ArrayList<>();
|
||||
|
||||
if (clusterPoints.size() <= sampleSize) {
|
||||
samplePoints = clusterPoints;
|
||||
} else {
|
||||
// Take evenly distributed samples
|
||||
int step = clusterPoints.size() / sampleSize;
|
||||
for (int i = 0; i < clusterPoints.size(); i += step) {
|
||||
samplePoints.add(clusterPoints.get(i));
|
||||
}
|
||||
}
|
||||
|
||||
// Use spatial grid approach to avoid distance calculations
|
||||
// Create a grid based on the bounding box of all points
|
||||
double minLat = clusterPoints.stream().mapToDouble(RawLocationPoint::getLatitude).min().orElse(0);
|
||||
double minLon = clusterPoints.stream().mapToDouble(RawLocationPoint::getLongitude).min().orElse(0);
|
||||
|
||||
// Grid cell size approximately 10 meters (rough approximation)
|
||||
double cellSizeLat = 0.0001; // ~11 meters
|
||||
double cellSizeLon = 0.0001; // varies by latitude but roughly 11 meters
|
||||
|
||||
// Create grid map for fast neighbor lookup
|
||||
Map<String, List<RawLocationPoint>> grid = new HashMap<>();
|
||||
for (RawLocationPoint point : clusterPoints) {
|
||||
int gridLat = (int) ((point.getLatitude() - minLat) / cellSizeLat);
|
||||
int gridLon = (int) ((point.getLongitude() - minLon) / cellSizeLon);
|
||||
String gridKey = gridLat + "," + gridLon;
|
||||
grid.computeIfAbsent(gridKey, _ -> new ArrayList<>()).add(point);
|
||||
}
|
||||
|
||||
RawLocationPoint bestPoint = null;
|
||||
double maxDensityScore = 0;
|
||||
|
||||
// Calculate density scores for sample points using grid lookup
|
||||
for (RawLocationPoint candidate : samplePoints) {
|
||||
double accuracy = candidate.getAccuracyMeters() != null && candidate.getAccuracyMeters() > 0
|
||||
? candidate.getAccuracyMeters()
|
||||
: 50.0;
|
||||
|
||||
// Calculate grid coordinates for candidate
|
||||
int candidateGridLat = (int) ((candidate.getLatitude() - minLat) / cellSizeLat);
|
||||
int candidateGridLon = (int) ((candidate.getLongitude() - minLon) / cellSizeLon);
|
||||
|
||||
// Search radius in grid cells (conservative estimate)
|
||||
int searchRadiusInCells = Math.max(1, (int) (accuracy / 100000)); // rough conversion
|
||||
|
||||
double densityScore = 0;
|
||||
|
||||
// Check neighboring grid cells
|
||||
for (int latOffset = -searchRadiusInCells; latOffset <= searchRadiusInCells; latOffset++) {
|
||||
for (int lonOffset = -searchRadiusInCells; lonOffset <= searchRadiusInCells; lonOffset++) {
|
||||
String neighborKey = (candidateGridLat + latOffset) + "," + (candidateGridLon + lonOffset);
|
||||
List<RawLocationPoint> neighbors = grid.get(neighborKey);
|
||||
|
||||
if (neighbors != null) {
|
||||
for (RawLocationPoint neighbor : neighbors) {
|
||||
if (candidate != neighbor) {
|
||||
// Simple proximity weight based on grid distance
|
||||
double gridDistance = Math.sqrt(latOffset * latOffset + lonOffset * lonOffset);
|
||||
double proximityWeight = Math.max(0, 1.0 - (gridDistance / searchRadiusInCells));
|
||||
densityScore += proximityWeight;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Combine density with accuracy weight
|
||||
double accuracyWeight = 1.0 / accuracy;
|
||||
densityScore = (densityScore * accuracyWeight) + accuracyWeight;
|
||||
|
||||
if (densityScore > maxDensityScore) {
|
||||
maxDensityScore = densityScore;
|
||||
bestPoint = candidate;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to first point if no best point found
|
||||
if (bestPoint == null) {
|
||||
bestPoint = clusterPoints.getFirst();
|
||||
}
|
||||
|
||||
return new GeoPoint(bestPoint.getLatitude(), bestPoint.getLongitude());
|
||||
}
|
||||
|
||||
private Trip createTripBetweenVisits(User user, String previewId,
|
||||
ProcessedVisit startVisit, ProcessedVisit endVisit) {
|
||||
// Trip starts when the first visit ends
|
||||
@@ -611,13 +861,13 @@ public class UnifiedLocationProcessingService {
|
||||
|
||||
// ==================== Result Classes ====================
|
||||
|
||||
private record VisitDetectionResult(List<Visit> visits, Instant searchStart, Instant searchEnd, long durationMillis) {
|
||||
private record VisitDetectionResult(List<Visit> visits, Instant searchStart, Instant searchEnd, long durationInMillis) {
|
||||
}
|
||||
|
||||
private record VisitMergingResult(List<Visit> inputVisits, List<ProcessedVisit> processedVisits,
|
||||
Instant searchStart, Instant searchEnd, long durationMillis) {
|
||||
Instant searchStart, Instant searchEnd, long durationInMillis) {
|
||||
}
|
||||
|
||||
private record TripDetectionResult(List<Trip> trips, long durationMillis) {
|
||||
private record TripDetectionResult(List<Trip> trips, long durationInMillis) {
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,7 +69,7 @@ reitti.security.oidc.registration.enabled=true
|
||||
|
||||
reitti.import.batch-size=10000
|
||||
# How many seconds should we wait after the last data input before starting to process all unprocessed data?
|
||||
reitti.import.processing-idle-start-time=15
|
||||
reitti.import.processing-idle-start-time=5
|
||||
|
||||
reitti.events.concurrency=1-16
|
||||
|
||||
@@ -105,7 +105,7 @@ reitti.storage.path=data/
|
||||
reitti.storage.cleanup.cron=0 0 4 * * *
|
||||
|
||||
# Location data density normalization
|
||||
reitti.location.density.target-points-per-minute=2
|
||||
reitti.location.density.target-points-per-minute=4
|
||||
|
||||
# Logging configuration
|
||||
reitti.logging.buffer-size=1000
|
||||
|
||||
@@ -236,11 +236,11 @@ label {
|
||||
|
||||
.toggle-buttons .btn {
|
||||
padding: 0.375rem 0.75rem;
|
||||
border: none;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s ease-in-out;
|
||||
font-size: 0.875rem;
|
||||
font-weight: 500;
|
||||
border: 1px solid var(--color-highlight);
|
||||
}
|
||||
|
||||
.toggle-buttons .btn:not(:last-child) {
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
hx-vals='js:{"timezone": getUserTimezone()}'
|
||||
hx-target="#preview-area"
|
||||
hx-include="#configuration-form"
|
||||
hx-trigger="change[target.value != '']"
|
||||
hx-trigger="change[target.value != ''] delay:1s"
|
||||
name="previewDate">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -13,7 +13,6 @@ import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.List;
|
||||
|
||||
import static com.dedicatedcode.reitti.TestConstants.Points.*;
|
||||
@@ -44,43 +43,40 @@ public class ProcessingPipelineTest {
|
||||
List<ProcessedVisit> processedVisits = currentVisits();
|
||||
assertEquals(5, processedVisits.size());
|
||||
|
||||
assertVisit(processedVisits.get(0), "2025-06-16T22:01:43Z","2025-06-17T05:41:30Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(1), "2025-06-17T05:46:11Z","2025-06-17T05:55:03Z" , new GeoPoint(53.86925378333333,10.711828311111113));
|
||||
assertVisit(processedVisits.get(2), "2025-06-17T06:00:30Z","2025-06-17T13:10:17Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(3), "2025-06-17T13:15:11Z","2025-06-17T13:19:22Z" , new GeoPoint(53.86753033888888,10.710380756666666));
|
||||
assertVisit(processedVisits.get(4), "2025-06-17T13:23:48Z","2025-06-17T21:59:44Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(0), "2025-06-16T22:00:09.154Z", "2025-06-17T05:40:26Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(1), "2025-06-17T05:43:37.962Z", "2025-06-17T05:55:03.792Z" , ST_THOMAS);
|
||||
assertVisit(processedVisits.get(2), "2025-06-17T05:58:10.797Z", "2025-06-17T13:08:53.346Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(3), "2025-06-17T13:12:01.542Z", "2025-06-17T13:18:51.590Z" , ST_THOMAS);
|
||||
assertVisit(processedVisits.get(4), "2025-06-17T13:21:28.334Z", "2025-06-17T21:59:44.876Z" , MOLTKESTR);
|
||||
|
||||
List<Trip> trips = currenTrips();
|
||||
assertEquals(4, trips.size());
|
||||
assertTrip(trips.get(0), "2025-06-17T05:41:30Z" , MOLTKESTR, "2025-06-17T05:46:11Z" , new GeoPoint(53.86925378333333,10.711828311111113));
|
||||
assertTrip(trips.get(1), "2025-06-17T05:55:03Z" , new GeoPoint(53.86925378333333,10.711828311111113), "2025-06-17T06:00:30Z" , MOLTKESTR);
|
||||
assertTrip(trips.get(2), "2025-06-17T13:10:17Z" , MOLTKESTR, "2025-06-17T13:15:11Z" , new GeoPoint(53.86753033888888,10.710380756666666));
|
||||
assertTrip(trips.get(3), "2025-06-17T13:19:22Z" , new GeoPoint(53.86753033888888,10.710380756666666), "2025-06-17T13:23:48Z" , MOLTKESTR);
|
||||
assertTrip(trips.get(0), "2025-06-17T05:40:26Z" , MOLTKESTR, "2025-06-17T05:43:37.962Z" , ST_THOMAS);
|
||||
assertTrip(trips.get(1), "2025-06-17T05:55:03.792Z" , ST_THOMAS, "2025-06-17T05:58:10.797Z" , MOLTKESTR);
|
||||
assertTrip(trips.get(2), "2025-06-17T13:08:53.346Z" , MOLTKESTR, "2025-06-17T13:12:01.542Z" , ST_THOMAS);
|
||||
assertTrip(trips.get(3), "2025-06-17T13:18:51.590Z" , ST_THOMAS, "2025-06-17T13:21:28.334Z" , MOLTKESTR);
|
||||
|
||||
testingService.importAndProcess(user, "/data/gpx/20250618.gpx");
|
||||
|
||||
processedVisits = currentVisits();
|
||||
|
||||
assertEquals(12, processedVisits.size());
|
||||
assertEquals(10, processedVisits.size());
|
||||
|
||||
|
||||
//new visits
|
||||
assertVisit(processedVisits.get(0), "2025-06-16T22:01:43Z", "2025-06-17T05:41:30Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(1), "2025-06-17T05:46:11Z", "2025-06-17T05:55:03Z", new GeoPoint(53.86925378333333,10.711828311111113));
|
||||
assertVisit(processedVisits.get(2), "2025-06-17T06:00:30Z", "2025-06-17T13:10:17Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(3), "2025-06-17T13:15:11Z", "2025-06-17T13:19:22Z", new GeoPoint(53.86753033888888,10.710380756666666));
|
||||
//should not touch visits before the new data
|
||||
assertVisit(processedVisits.get(0), "2025-06-16T22:00:09.154Z", "2025-06-17T05:41:00Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(1), "2025-06-17T05:41:30.989Z", "2025-06-17T05:57:07.729Z" , ST_THOMAS);
|
||||
assertVisit(processedVisits.get(2), "2025-06-17T05:57:41Z" , "2025-06-17T13:09:29Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(3), "2025-06-17T13:09:51.476Z", "2025-06-17T13:20:24.494Z" , ST_THOMAS);
|
||||
|
||||
//should extend the last visit of the old day
|
||||
assertVisit(processedVisits.get(4), "2025-06-17T13:23:48Z" ,"2025-06-18T05:47:44Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(4), "2025-06-17T13:20:58Z" , "2025-06-18T05:46:43Z", MOLTKESTR);
|
||||
|
||||
//should not touch visits after the new data
|
||||
assertVisit(processedVisits.get(5) ,"2025-06-18T05:48:15Z" ,"2025-06-18T05:53:33Z", new GeoPoint(53.863864, 10.708570));
|
||||
assertVisit(processedVisits.get(6) ,"2025-06-18T05:56:46Z" ,"2025-06-18T06:03:00Z", new GeoPoint(53.86753033888888,10.710380756666666));
|
||||
assertVisit(processedVisits.get(7) ,"2025-06-18T06:07:46Z" ,"2025-06-18T13:02:27Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(8) ,"2025-06-18T13:07:42Z" ,"2025-06-18T13:16:57Z", new GeoPoint(53.86925378333333,10.711828311111113));
|
||||
assertVisit(processedVisits.get(9) ,"2025-06-18T13:20:10Z" ,"2025-06-18T13:29:46Z", new GeoPoint(53.872904, 10.720157));
|
||||
assertVisit(processedVisits.get(10),"2025-06-18T13:35:46Z" ,"2025-06-18T15:52:19Z", GARTEN);
|
||||
assertVisit(processedVisits.get(11),"2025-06-18T16:05:49Z" ,"2025-06-18T21:59:29Z", MOLTKESTR);
|
||||
//new visits
|
||||
assertVisit(processedVisits.get(5), "2025-06-18T05:47:13.682Z" ,"2025-06-18T06:04:02.435Z" , ST_THOMAS);
|
||||
assertVisit(processedVisits.get(6), "2025-06-18T06:04:36Z" ,"2025-06-18T13:01:57Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(7), "2025-06-18T13:02:27.656Z" ,"2025-06-18T13:14:19.417Z" , ST_THOMAS);
|
||||
assertVisit(processedVisits.get(8), "2025-06-18T13:33:05Z" ,"2025-06-18T15:50:40Z" , GARTEN);
|
||||
assertVisit(processedVisits.get(9), "2025-06-18T16:02:38Z" ,"2025-06-18T21:59:29.055Z" , MOLTKESTR);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -88,40 +84,36 @@ public class ProcessingPipelineTest {
|
||||
testingService.importAndProcess(user, "/data/gpx/20250618.gpx");
|
||||
|
||||
List<ProcessedVisit> processedVisits = currentVisits();
|
||||
assertEquals(8, processedVisits.size());
|
||||
assertEquals(6, processedVisits.size());
|
||||
|
||||
assertVisit(processedVisits.get(0), "2025-06-17T22:01:48Z" ,"2025-06-18T05:47:44Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(1), "2025-06-18T05:48:15Z" ,"2025-06-18T05:53:33Z", new GeoPoint(53.863864, 10.708570));
|
||||
assertVisit(processedVisits.get(2), "2025-06-18T05:56:46Z" ,"2025-06-18T06:03:00Z", ST_THOMAS);
|
||||
assertVisit(processedVisits.get(3), "2025-06-18T06:07:46Z" ,"2025-06-18T13:02:27Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(4), "2025-06-18T13:07:42Z" ,"2025-06-18T13:16:57Z", ST_THOMAS);
|
||||
assertVisit(processedVisits.get(5), "2025-06-18T13:20:10Z" ,"2025-06-18T13:29:46Z", new GeoPoint(53.872904, 10.720157));
|
||||
assertVisit(processedVisits.get(6), "2025-06-18T13:35:46Z" ,"2025-06-18T15:52:19Z", GARTEN);
|
||||
assertVisit(processedVisits.get(7), "2025-06-18T16:05:49Z" ,"2025-06-18T21:59:29Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(0), "2025-06-17T22:00:15.843Z" ,"2025-06-18T05:46:10Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(1), "2025-06-18T05:53:33.667Z" ,"2025-06-18T06:01:54.440Z" , ST_THOMAS);
|
||||
assertVisit(processedVisits.get(2), "2025-06-18T06:04:36Z" ,"2025-06-18T13:01:57Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(3), "2025-06-18T13:04:33.424Z" ,"2025-06-18T13:13:47.443Z" , ST_THOMAS);
|
||||
assertVisit(processedVisits.get(4), "2025-06-18T13:33:35.626Z" ,"2025-06-18T15:50:40Z" , GARTEN);
|
||||
assertVisit(processedVisits.get(5), "2025-06-18T16:02:38Z" ,"2025-06-18T21:59:29.055Z" , MOLTKESTR);
|
||||
|
||||
testingService.importAndProcess(user, "/data/gpx/20250617.gpx");
|
||||
|
||||
processedVisits = currentVisits();
|
||||
|
||||
assertEquals(12, processedVisits.size());
|
||||
assertEquals(10, processedVisits.size());
|
||||
|
||||
//new visits
|
||||
assertVisit(processedVisits.get(0), "2025-06-16T22:01:43Z", "2025-06-17T05:41:30Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(1), "2025-06-17T05:46:11Z", "2025-06-17T05:55:03Z", ST_THOMAS);
|
||||
assertVisit(processedVisits.get(2), "2025-06-17T06:00:30Z", "2025-06-17T13:10:17Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(3), "2025-06-17T13:15:11Z", "2025-06-17T13:19:22Z", ST_THOMAS);
|
||||
assertVisit(processedVisits.get(0), "2025-06-16T22:00:09.154Z", "2025-06-17T05:41:00Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(1), "2025-06-17T05:41:30.989Z", "2025-06-17T05:57:07.729Z" , ST_THOMAS);
|
||||
assertVisit(processedVisits.get(2), "2025-06-17T05:57:41Z" , "2025-06-17T13:09:29Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(3), "2025-06-17T13:09:51.476Z", "2025-06-17T13:20:24.494Z" , ST_THOMAS);
|
||||
|
||||
//should extend the last visit of the old day
|
||||
assertVisit(processedVisits.get(4), "2025-06-17T13:23:48Z" ,"2025-06-18T05:47:44Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(4), "2025-06-17T13:20:58Z" , "2025-06-18T05:46:43Z" , MOLTKESTR);
|
||||
|
||||
//should not touch visits after the new data
|
||||
assertVisit(processedVisits.get(5) ,"2025-06-18T05:48:15Z" ,"2025-06-18T05:53:33Z", new GeoPoint(53.863864, 10.708570));
|
||||
assertVisit(processedVisits.get(6) ,"2025-06-18T05:56:46Z" ,"2025-06-18T06:03:00Z", ST_THOMAS);
|
||||
assertVisit(processedVisits.get(7) ,"2025-06-18T06:07:46Z" ,"2025-06-18T13:02:27Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(8) ,"2025-06-18T13:07:42Z" ,"2025-06-18T13:16:57Z", ST_THOMAS);
|
||||
assertVisit(processedVisits.get(9) ,"2025-06-18T13:20:10Z" ,"2025-06-18T13:29:46Z", new GeoPoint(53.872904, 10.720157));
|
||||
assertVisit(processedVisits.get(10),"2025-06-18T13:35:46Z" ,"2025-06-18T15:52:19Z", GARTEN);
|
||||
assertVisit(processedVisits.get(11),"2025-06-18T16:05:49Z" ,"2025-06-18T21:59:29Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(5), "2025-06-18T05:47:13.682Z" ,"2025-06-18T06:04:02.435Z" , ST_THOMAS);
|
||||
assertVisit(processedVisits.get(6), "2025-06-18T06:04:36Z" ,"2025-06-18T13:01:57Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(7), "2025-06-18T13:02:27.656Z" ,"2025-06-18T13:14:19.417Z" , ST_THOMAS);
|
||||
assertVisit(processedVisits.get(8), "2025-06-18T13:33:05Z" ,"2025-06-18T15:50:40Z" , GARTEN);
|
||||
assertVisit(processedVisits.get(9), "2025-06-18T16:02:38Z" ,"2025-06-18T21:59:29.055Z" , MOLTKESTR);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -129,16 +121,14 @@ public class ProcessingPipelineTest {
|
||||
testingService.importAndProcess(user, "/data/gpx/20250618.gpx");
|
||||
|
||||
List<ProcessedVisit> processedVisits = currentVisits();
|
||||
assertEquals(8, processedVisits.size());
|
||||
assertEquals(6, processedVisits.size());
|
||||
|
||||
assertVisit(processedVisits.get(0), "2025-06-17T22:01:48Z", "2025-06-18T05:47:44Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(1), "2025-06-18T05:48:15Z", "2025-06-18T05:53:33Z", new GeoPoint(53.86386352818182,10.708570236363638));
|
||||
assertVisit(processedVisits.get(2), "2025-06-18T05:56:46Z", "2025-06-18T06:03:00Z", ST_THOMAS);
|
||||
assertVisit(processedVisits.get(3), "2025-06-18T06:07:46Z", "2025-06-18T13:02:27Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(4), "2025-06-18T13:07:42Z", "2025-06-18T13:16:57Z", new GeoPoint(53.868274,10.712731));
|
||||
assertVisit(processedVisits.get(5), "2025-06-18T13:20:10Z", "2025-06-18T13:29:46Z", new GeoPoint(53.872904,10.720157));
|
||||
assertVisit(processedVisits.get(6), "2025-06-18T13:35:46Z", "2025-06-18T15:52:19Z", GARTEN);
|
||||
assertVisit(processedVisits.get(7), "2025-06-18T16:05:49Z", "2025-06-18T21:59:29Z", MOLTKESTR);
|
||||
assertVisit(processedVisits.get(0), "2025-06-17T22:00:15.843Z" , "2025-06-18T05:46:10Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(1), "2025-06-18T05:53:33.667Z" ,"2025-06-18T06:01:54.440Z" , ST_THOMAS);
|
||||
assertVisit(processedVisits.get(2), "2025-06-18T06:04:36Z" ,"2025-06-18T13:01:57Z" , MOLTKESTR);
|
||||
assertVisit(processedVisits.get(3), "2025-06-18T13:04:33.424Z" ,"2025-06-18T13:13:47.443Z" , ST_THOMAS);
|
||||
assertVisit(processedVisits.get(4), "2025-06-18T13:33:35.626Z" ,"2025-06-18T15:50:40Z" , GARTEN);
|
||||
assertVisit(processedVisits.get(5), "2025-06-18T16:02:38Z" ,"2025-06-18T21:59:29.055Z" , MOLTKESTR);
|
||||
}
|
||||
|
||||
|
||||
@@ -184,15 +174,15 @@ public class ProcessingPipelineTest {
|
||||
for (int i = 0; i < processedVisitsOutOfOrder.size(); i++) {
|
||||
ProcessedVisit processedVisit = processedVisitsOutOfOrder.get(i);
|
||||
ProcessedVisit processedVisitInOrder = processedVisitsInOrder.get(i);
|
||||
assertEquals(processedVisitInOrder.getStartTime().truncatedTo(ChronoUnit.SECONDS), processedVisit.getStartTime().truncatedTo(ChronoUnit.SECONDS));
|
||||
assertEquals(processedVisitInOrder.getEndTime().truncatedTo(ChronoUnit.SECONDS), processedVisit.getEndTime().truncatedTo(ChronoUnit.SECONDS));
|
||||
assertEquals(processedVisitInOrder.getStartTime(), processedVisit.getStartTime());
|
||||
assertEquals(processedVisitInOrder.getEndTime(), processedVisit.getEndTime());
|
||||
assertEquals(processedVisitInOrder.getPlace(), processedVisit.getPlace());
|
||||
}
|
||||
|
||||
}
|
||||
private static void assertVisit(ProcessedVisit processedVisit, String startTime, String endTime, GeoPoint location) {
|
||||
assertEquals(Instant.parse(startTime).truncatedTo(ChronoUnit.SECONDS), processedVisit.getStartTime().truncatedTo(ChronoUnit.SECONDS));
|
||||
assertEquals(Instant.parse(endTime).truncatedTo(ChronoUnit.SECONDS), processedVisit.getEndTime().truncatedTo(ChronoUnit.SECONDS));
|
||||
assertEquals(Instant.parse(startTime), processedVisit.getStartTime());
|
||||
assertEquals(Instant.parse(endTime), processedVisit.getEndTime());
|
||||
GeoPoint currentLocation = new GeoPoint(processedVisit.getPlace().getLatitudeCentroid(), processedVisit.getPlace().getLongitudeCentroid());
|
||||
assertTrue(location.near(currentLocation), "Locations are not near to each other. \nExpected [" + currentLocation + "] to be in range \nto [" + location + "]");
|
||||
}
|
||||
@@ -206,8 +196,8 @@ public class ProcessingPipelineTest {
|
||||
}
|
||||
|
||||
private static void assertTrip(Trip trip, String startTime, GeoPoint startLocation, String endTime, GeoPoint endLocation) {
|
||||
assertEquals(Instant.parse(startTime).truncatedTo(ChronoUnit.SECONDS), trip.getStartTime().truncatedTo(ChronoUnit.SECONDS));
|
||||
assertEquals(Instant.parse(endTime).truncatedTo(ChronoUnit.SECONDS), trip.getEndTime().truncatedTo(ChronoUnit.SECONDS));
|
||||
assertEquals(Instant.parse(startTime), trip.getStartTime());
|
||||
assertEquals(Instant.parse(endTime), trip.getEndTime());
|
||||
|
||||
GeoPoint actualStartLocation = GeoPoint.from(trip.getStartVisit().getPlace().getLatitudeCentroid(), trip.getStartVisit().getPlace().getLongitudeCentroid());
|
||||
assertTrue(startLocation.near(actualStartLocation),
|
||||
|
||||
Reference in New Issue
Block a user