diff --git a/.app_version b/.app_version index aa22d3ce..54d1a4f2 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.12.3 +0.13.0 diff --git a/.dockerignore b/.dockerignore index 25ac0a6f..fc5c0193 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,8 @@ /log /tmp + +# We need directories for import and export files, but not the files themselves. +/public/exports/* +!/public/exports/.keep +/public/imports/* +!/public/imports/.keep diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f5ad770..00b9be29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,42 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [0.13.0] — 2024-09-03 + +⚠️ BREAKING CHANGES: ⚠️ + +Default exporting format is now GeoJSON instead of Owntracks-like JSON. This will allow you to use the exported data in other applications that support GeoJSON format. It's also important to highlight, that GeoJSON format does not describe a way to store any time-related data. Dawarich relies on the `timestamp` field in the GeoJSON format to determine the time of the point. The value of the `timestamp` field should be a Unix timestamp in seconds. If you import GeoJSON data that does not have a `timestamp` field, the point will not be imported. + +Example of a valid point in GeoJSON format: + +```json +{ + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [13.350110811262352, 52.51450815] + }, + "properties": { + "timestamp": 1725310036 + } +} +``` + +### Added + +- GeoJSON format is now available for exporting data. +- GPX format is now available for exporting data. +- Importing GeoJSON is now available. + +### Changed + +- Default exporting format is now GeoJSON instead of Owntracks-like JSON. This will allow you to use the exported data in other applications that support GeoJSON format. + +### Fixed + +- Fixed a bug where the confirmation alert was shown more than once when deleting a point. + + ## [0.12.3] — 2024-09-02 ### Added diff --git a/Gemfile b/Gemfile index e4e3c97f..1da98ac5 100644 --- a/Gemfile +++ b/Gemfile @@ -9,6 +9,7 @@ gem 'chartkick' gem 'data_migrate' gem 'devise' gem 'geocoder' +gem 'gpx' gem 'httparty' gem 'importmap-rails' gem 'kaminari' diff --git a/Gemfile.lock b/Gemfile.lock index b1e642b8..b1e5dda4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -137,6 +137,9 @@ GEM csv (>= 3.0.0) globalid (1.2.1) activesupport (>= 6.1) + gpx (1.1.1) + nokogiri (~> 1.7) + rake hashdiff (1.1.0) httparty (0.22.0) csv @@ -433,6 +436,7 @@ DEPENDENCIES ffaker foreman geocoder + gpx httparty importmap-rails kaminari diff --git a/app/controllers/exports_controller.rb b/app/controllers/exports_controller.rb index 2f7110ef..23af2622 100644 --- a/app/controllers/exports_controller.rb +++ b/app/controllers/exports_controller.rb @@ -12,7 +12,7 @@ class ExportsController < ApplicationController export_name = "export_from_#{params[:start_at].to_date}_to_#{params[:end_at].to_date}" export = current_user.exports.create(name: export_name, status: :created) - ExportJob.perform_later(export.id, params[:start_at], params[:end_at]) + ExportJob.perform_later(export.id, params[:start_at], params[:end_at], file_format: params[:file_format]) redirect_to exports_url, notice: 'Export was successfully initiated. Please wait until it\'s finished.' rescue StandardError => e diff --git a/app/javascript/controllers/maps_controller.js b/app/javascript/controllers/maps_controller.js index 4a1e82cc..1e562441 100644 --- a/app/javascript/controllers/maps_controller.js +++ b/app/javascript/controllers/maps_controller.js @@ -145,8 +145,12 @@ export default class extends Controller { `; } + removeEventListeners() { + document.removeEventListener('click', this.handleDeleteClick); + } + addEventListeners() { - document.addEventListener('click', (event) => { + this.handleDeleteClick = (event) => { if (event.target && event.target.classList.contains('delete-point')) { event.preventDefault(); const pointId = event.target.getAttribute('data-id'); @@ -155,7 +159,11 @@ export default class extends Controller { this.deletePoint(pointId, this.apiKey); } } - }); + }; + + // Ensure only one listener is attached by removing any existing ones first + this.removeEventListeners(); + document.addEventListener('click', this.handleDeleteClick); } deletePoint(id, apiKey) { diff --git a/app/jobs/export_job.rb b/app/jobs/export_job.rb index 0aadd35a..b8872c05 100644 --- a/app/jobs/export_job.rb +++ b/app/jobs/export_job.rb @@ -3,9 +3,9 @@ class ExportJob < ApplicationJob queue_as :exports - def perform(export_id, start_at, end_at) + def perform(export_id, start_at, end_at, file_format: :json) export = Export.find(export_id) - Exports::Create.new(export:, start_at:, end_at:).call + Exports::Create.new(export:, start_at:, end_at:, file_format:).call end end diff --git a/app/jobs/import_job.rb b/app/jobs/import_job.rb index f421b927..5a13d76d 100644 --- a/app/jobs/import_job.rb +++ b/app/jobs/import_job.rb @@ -32,6 +32,7 @@ class ImportJob < ApplicationJob when 'owntracks' then OwnTracks::ExportParser when 'gpx' then Gpx::TrackParser when 'immich_api' then Immich::ImportParser + when 'geojson' then Geojson::ImportParser end end diff --git a/app/models/import.rb b/app/models/import.rb index 332eec18..133f1783 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -10,6 +10,6 @@ class Import < ApplicationRecord enum source: { google_semantic_history: 0, owntracks: 1, google_records: 2, - google_phone_takeout: 3, gpx: 4, immich_api: 5 + google_phone_takeout: 3, gpx: 4, immich_api: 5, geojson: 6 } end diff --git a/app/serializers/point_serializer.rb b/app/serializers/point_serializer.rb new file mode 100644 index 00000000..4bbd7ad0 --- /dev/null +++ b/app/serializers/point_serializer.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class PointSerializer + EXCLUDED_ATTRIBUTES = %w[created_at updated_at visit_id id import_id user_id].freeze + + def initialize(point) + @point = point + end + + def call + point.attributes.except(*EXCLUDED_ATTRIBUTES) + end + + private + + attr_reader :point +end diff --git a/app/serializers/points/geojson_serializer.rb b/app/serializers/points/geojson_serializer.rb new file mode 100644 index 00000000..40c1048f --- /dev/null +++ b/app/serializers/points/geojson_serializer.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class Points::GeojsonSerializer + def initialize(points) + @points = points + end + + # rubocop:disable Metrics/MethodLength + def call + { + type: 'FeatureCollection', + features: points.map do |point| + { + type: 'Feature', + geometry: { + type: 'Point', + coordinates: [point.longitude, point.latitude] + }, + properties: PointSerializer.new(point).call + } + end + }.to_json + end + # rubocop:enable Metrics/MethodLength + + private + + attr_reader :points +end diff --git a/app/serializers/points/gpx_serializer.rb b/app/serializers/points/gpx_serializer.rb new file mode 100644 index 00000000..c52c1e9b --- /dev/null +++ b/app/serializers/points/gpx_serializer.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class Points::GpxSerializer + def initialize(points) + @points = points + end + + def call + geojson_data = Points::GeojsonSerializer.new(points).call + + GPX::GeoJSON.convert_to_gpx(geojson_data:) + end + + private + + attr_reader :points +end diff --git a/app/services/exports/create.rb b/app/services/exports/create.rb index 0024aa6c..c7c01718 100644 --- a/app/services/exports/create.rb +++ b/app/services/exports/create.rb @@ -1,33 +1,27 @@ # frozen_string_literal: true class Exports::Create - def initialize(export:, start_at:, end_at:) - @export = export - @user = export.user - @start_at = start_at.to_datetime - @end_at = end_at.to_datetime + def initialize(export:, start_at:, end_at:, file_format: :json) + @export = export + @user = export.user + @start_at = start_at.to_datetime + @end_at = end_at.to_datetime + @file_format = file_format end def call export.update!(status: :processing) - Rails.logger.debug "====Exporting data for #{user.email} from #{start_at} to #{end_at}" - points = time_framed_points - Rails.logger.debug "====Exporting #{points.size} points" + data = points_data(points) - data = ::ExportSerializer.new(points, user.email).call - file_path = Rails.root.join('public', 'exports', "#{export.name}.json") + create_export_file(data) - File.open(file_path, 'w') { |file| file.write(data) } - - export.update!(status: :completed, url: "exports/#{export.name}.json") + export.update!(status: :completed, url: "exports/#{export.name}.#{file_format}") create_export_finished_notification rescue StandardError => e - Rails.logger.error("====Export failed to create: #{e.message}") - create_failed_export_notification(e) export.update!(status: :failed) @@ -35,7 +29,7 @@ class Exports::Create private - attr_reader :user, :export, :start_at, :end_at + attr_reader :user, :export, :start_at, :end_at, :file_format def time_framed_points user @@ -60,4 +54,26 @@ class Exports::Create content: "Export \"#{export.name}\" failed: #{error.message}, stacktrace: #{error.backtrace.join("\n")}" ).call end + + def points_data(points) + case file_format.to_sym + when :json then process_geojson_export(points) + when :gpx then process_gpx_export(points) + else raise ArgumentError, "Unsupported file format: #{file_format}" + end + end + + def process_geojson_export(points) + Points::GeojsonSerializer.new(points).call + end + + def process_gpx_export(points) + Points::GpxSerializer.new(points).call + end + + def create_export_file(data) + file_path = Rails.root.join('public', 'exports', "#{export.name}.#{file_format}") + + File.open(file_path, 'w') { |file| file.write(data) } + end end diff --git a/app/services/geojson/import_parser.rb b/app/services/geojson/import_parser.rb new file mode 100644 index 00000000..ba3d333f --- /dev/null +++ b/app/services/geojson/import_parser.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class Geojson::ImportParser + attr_reader :import, :json, :user_id + + def initialize(import, user_id) + @import = import + @json = import.raw_data + @user_id = user_id + end + + def call + data = Geojson::Params.new(json).call + + data.each do |point| + next if point_exists?(point, user_id) + + Point.create!(point.merge(user_id:, import_id: import.id)) + end + end + + private + + def point_exists?(params, user_id) + Point.exists?( + latitude: params[:latitude], + longitude: params[:longitude], + timestamp: params[:timestamp], + user_id: + ) + end +end diff --git a/app/services/geojson/params.rb b/app/services/geojson/params.rb new file mode 100644 index 00000000..c87ad072 --- /dev/null +++ b/app/services/geojson/params.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +class Geojson::Params + attr_reader :json + + def initialize(json) + @json = json.with_indifferent_access + end + + def call + json['features'].map do |point| + next if point[:geometry].nil? || point.dig(:properties, :timestamp).nil? + + { + latitude: point[:geometry][:coordinates][1], + longitude: point[:geometry][:coordinates][0], + battery_status: point[:properties][:battery_state], + battery: battery_level(point[:properties][:battery_level]), + timestamp: Time.zone.at(point[:properties][:timestamp]), + altitude: point[:properties][:altitude], + velocity: point[:properties][:speed], + tracker_id: point[:properties][:device_id], + ssid: point[:properties][:wifi], + accuracy: point[:properties][:horizontal_accuracy], + vertical_accuracy: point[:properties][:vertical_accuracy], + raw_data: point + } + end.compact + end + + private + + def battery_level(level) + value = (level.to_f * 100).to_i + + value.positive? ? value : nil + end +end diff --git a/app/views/imports/_form.html.erb b/app/views/imports/_form.html.erb index e5684a7f..83797c82 100644 --- a/app/views/imports/_form.html.erb +++ b/app/views/imports/_form.html.erb @@ -22,6 +22,15 @@

A JSON file you exported by pressing Download button in top right corner of OwnTracks web interface

+
+
+ +

A valid GeoJSON file. For example, a file, exported from a Dawarich instance

+
+