Parsovani GPX

.env.dist

@@ -6,3 +6,4 @@ MYSQL_HOST: "127.0.0.1"
 MYSQL_DATABASE: "poletani_cz"
 MYSQL_USER: "mysql_user"
 MYSQL_PASSWORD: "****"
+APP_ENV: "production"

docker-compose.prod.yml

@@ -1,7 +1,5 @@
 version: "3.3"
 
-# testovaci koment
-
 services:
   api:
     image: "docker.kvacek.cz/poletani/api:latest"

docker-compose.yml

@@ -20,8 +20,10 @@ services:
     environment:
       TZ: "Europe/Prague"
       API_URL: "http://localhost:8000"
+      APP_ENV: "development"
       ALLOW_CORS_ORIGINS: "http://localhost:9000 http://localhost:9001"
 
+
   scrapy:
     build:
       context: ../crawlers

src/database/config.py

@@ -14,7 +14,7 @@ def get_database_url():
 
 def create_db_engine():
     database_url = get_database_url()
-    return create_async_engine(database_url, future=True, echo=True)
+    return create_async_engine(database_url, future=True, echo=True, pool_pre_ping=True)
 
 
 engine = create_db_engine()

src/external/elevation.py

@@ -0,0 +1,25 @@
+from typing import List, Tuple, Dict
+
+import aiohttp
+
+
+class ElevationAPI:
+    ELEVATION_ENDPOINT = "https://api.open-elevation.com/api/v1/lookup"
+
+    def get_request(self, points: List[Dict[str, float]]):
+        return {"locations": [{"latitude": point['lat'], "longitude": point['lng']} for point in points]}
+
+    async def call_api(self, points: List[Dict[str, float]]):
+        async with aiohttp.ClientSession() as session:
+            async with session.post(self.ELEVATION_ENDPOINT, json=self.get_request(points)) as resp:
+                resp.raise_for_status()
+                return await resp.json()
+
+    async def get_elevation_for_points(self, points: List[Dict[str, float]]) -> List[dict]:
+        response = await self.call_api(points)
+
+        return [{
+            "lat": loc['latitude'],
+            "lng": loc['longitude'],
+            "elevation": loc['elevation']
+        } for loc in response['results']]

src/external/gpx_parser.py

@@ -0,0 +1,93 @@
+from datetime import datetime
+from typing import List, Dict
+
+from aiocache import cached
+from lxml import etree
+from lxml.etree import _ElementTree
+
+
+class GPXParser:
+    def __init__(self, file: str):
+        self.file = file
+        self.gpx = etree.parse(self.file)
+        self.namespace = None
+        self.precision_digits = 6
+        self.set_namespace()
+
+    def set_namespace(self):
+        namespace = self.gpx.getroot().nsmap.get(None)
+        self.namespace = {'gpx': namespace}
+
+    def run_xpath(self, path: str):
+        return self.gpx.xpath(path, namespaces=self.namespace)
+
+    @cached()
+    async def get_times(self):
+        nodes = self.run_xpath("//gpx:time")
+        return [datetime.fromisoformat(node.text).astimezone() for node in nodes]
+
+    @cached()
+    async def get_coordinates(self) -> List[Dict[str, float]]:
+        nodes = self.run_xpath("//gpx:trkpt")
+        return [{"lat": float(node.attrib["lat"]), "lng": float(node.attrib['lon'])} for node in nodes]
+
+    @cached()
+    async def get_speed(self) -> List[int]:
+        nodes = self.run_xpath("//gpx:speed")
+        return [int(node.text) for node in nodes]
+
+    @cached()
+    async def get_magnetic_variation(self) -> List[int]:
+        nodes = self.run_xpath("//gpx:magvar")
+        return [int(node.text) for node in nodes]
+
+    @cached()
+    async def get_altitude(self) -> List[float]:
+        nodes = self.run_xpath("//gpx:ele")
+        return [float(node.text) for node in nodes]
+
+    @cached()
+    async def get_terrain_elevation(self) -> List[float]:
+        nodes = self.run_xpath("//gpx:terrain_elevation")
+        return [float(node.text) for node in nodes]
+
+    @cached()
+    async def get_max_speed(self):
+        return max(await self.get_speed())
+
+    @cached()
+    async def get_avg_speed(self):
+        speeds = await self.get_speed()
+        return round(sum(speeds) / len(speeds), 2)
+
+    @cached()
+    async def get_max_altitude(self):
+        return max(await self.get_altitude())
+
+    @cached()
+    async def get_avg_altitude(self):
+        altitudes = await self.get_altitude()
+        return round(sum(altitudes) / len(altitudes), 2)
+
+    def add_terrain_elevation(self, points_with_elevation: List[Dict[str, float]]):
+        track_points = self.run_xpath("//gpx:trkpt")
+        # TODO: open elevation API umi jen presnost na 6 desetinnych mist!
+        track_points_index = {(float(n.attrib['lat']), float(n.attrib['lon'])): n for n in track_points}
+
+        for point_with_elevation in points_with_elevation:
+            lat = point_with_elevation['lat']
+            lng = point_with_elevation['lng']
+            elevation = point_with_elevation['elevation']
+            target_node = track_points_index.get((lat, lng))
+
+            if target_node is None:
+                continue
+
+            extensions = target_node.find("./extensions", self.gpx.getroot().nsmap)
+            extensions.append(etree.XML(f"<terrain_elevation>{elevation}</terrain_elevation>"))
+
+        return self.gpx
+
+    def write(self, tree: _ElementTree, output: str):
+        print(f"ZAPISUJI DO {output}")
+        tree.write(output)

src/gpx_test.py

@@ -0,0 +1,13 @@
+import asyncio
+from external.elevation import ElevationAPI
+from external.gpx_parser import GPXParser
+
+
+async def test():
+    elevation_api = ElevationAPI()
+    parser = GPXParser("./uploads/tracks/37b979cb-71c0-4d09-a2c1-cfdad5f7a0cf-OK-AUR_28_AUR_Bristell_NG5_Zapisnik_letu_2023-08-04-00 00_2023-08-04-12 00(1).gpx")
+    points = await parser.get_coordinates()
+    elevation = await elevation_api.get_elevation_for_points(points)
+    parser.add_terrain_elevation(elevation)
+
+asyncio.run(test())

src/graphql_schema/entities/flight.py

@@ -1,5 +1,5 @@
 import asyncio
-from datetime import timedelta
+from datetime import timedelta, datetime
 from typing import List, Optional, Annotated, TYPE_CHECKING, Tuple
 import strawberry
 from fastapi import HTTPException
@@ -11,6 +11,7 @@ from database import models
 from database.models import flight_has_copilot
 from decorators.endpoints import authenticated_user_only
 from decorators.error_logging import error_logging
+from external.gpx_parser import GPXParser
 from graphql_schema.dataloaders.aircraft import aircraft_dataloader
 from graphql_schema.dataloaders.airport import airport_dataloader
 from graphql_schema.dataloaders.copilots import flight_copilots_dataloader
@@ -23,7 +24,7 @@ from graphql_schema.entities.photo import Photo
 from graphql_schema.entities.poi import PointOfInterest
 from graphql_schema.sqlalchemy_to_strawberry_type import strawberry_sqlalchemy_type, strawberry_sqlalchemy_input
 from upload_utils import get_public_url
-from .helpers.flight import handle_aircraft_save, handle_track_edit, handle_copilots_edit, handle_weather_info, get_airports, handle_upload_gpx, handle_airport_changed
+from .helpers.flight import handle_aircraft_save, handle_track_edit, handle_copilots_edit, handle_weather_info, get_airports, handle_upload_gpx, handle_airport_changed, add_terrain_elevation
 from ..types import ComboboxInput
 
 if TYPE_CHECKING:
@@ -51,10 +52,16 @@ class Point:
 
 @strawberry.type
 class GPXTrack:
-    points: List[Point]
+    coordinates: List[Point]
     speed: List[float]
-    elevation: List[float]
+    altitude: List[float]
+    magnetic_variation: List[float]
     terrain_elevation: List[float]
+    time: List[datetime]
+    max_speed: float
+    avg_speed: float
+    max_altitude: float
+    avg_altitude: float
 
 
 @strawberry_sqlalchemy_type(models.Flight)
@@ -97,18 +104,22 @@ class Flight:
         if not root.gpx_track_filename:
             return None
 
-        path = "/app/uploads/tracks"
+        try:
+            gpx_parser = GPXParser(f"/app/uploads/tracks/{root.gpx_track_filename}")
+        except OSError:
+            return None
 
-        tree = etree.parse(f"{path}/{root.gpx_track_filename}")
-        points = tree.findall("//trk")
-        speed = tree.xpath("//extensions/speed")
-        elevation = tree.xpath('//ele')
-        print(tree.getroot().find("trk"), list(points), speed, elevation)
         return GPXTrack(
-            points=[Point(lat=50.123, lng=14.324243)],
-            speed=[],
-            elevation=[],
-            terrain_elevation=[]
+            coordinates=[Point(**point) for point in await gpx_parser.get_coordinates()],
+            speed=await gpx_parser.get_speed(),
+            altitude=await gpx_parser.get_altitude(),
+            terrain_elevation=await gpx_parser.get_terrain_elevation(),
+            time=await gpx_parser.get_times(),
+            max_speed=await gpx_parser.get_max_speed(),
+            avg_speed=await gpx_parser.get_avg_speed(),
+            max_altitude=await gpx_parser.get_max_altitude(),
+            avg_altitude=await gpx_parser.get_avg_altitude(),
+            magnetic_variation=await gpx_parser.get_magnetic_variation(),
         )
 
     def load_gpx_track_url(root):
@@ -131,7 +142,7 @@ class Flight:
     takeoff_weather_info: Optional[WeatherInfo] = strawberry.field(resolver=load_takeoff_weather_info)
     landing_weather_info: Optional[WeatherInfo] = strawberry.field(resolver=load_landing_weather_info)
     photos: List[Photo] = strawberry.field(resolver=load_photos)
-    gpx_track_url: Optional[str] = strawberry.field(resolver=load_gpx_track_url)
+    gpx_track_url: Optional[str] = strawberry.field(resolver=load_gpx_track_url)  # TODO: odstranit
     gpx_track: Optional[GPXTrack] = strawberry.field(resolver=load_gpx_track)
 
 
@@ -222,8 +233,8 @@ class CreateFlightMutation:
 @strawberry.type
 class EditFlightMutation:
     @strawberry_sqlalchemy_input(models.Flight, exclude_fields=[
-        "id", "aircraft_id", "deleted", "landing_airport_id", "takeoff_airport_id"
-                                                              "takeoff_weather_info_id", "landing_weather_info_id", "gpx_track_filename"
+        "id", "aircraft_id", "deleted", "landing_airport_id", "takeoff_airport_id",
+        "takeoff_weather_info_id", "landing_weather_info_id", "gpx_track_filename"
     ], all_optional=True)
     class EditFlightInput:
         gpx_track: Optional[Upload] = None  # TODO: poresit validaci uploadovaneho souboru!
@@ -253,7 +264,9 @@ class EditFlightMutation:
 
         if input.gpx_track is not None:
             data['gpx_track_filename'] = await handle_upload_gpx(flight, input.gpx_track)
+            info.context.background_tasks.add_task(add_terrain_elevation, flight=flight, gpx_filename=data['gpx_track_filename'], db=db)
 
+        # TODO: nasledujici metody volat i pokud se zmenil cas vzletu!
         if input.takeoff_airport and input.takeoff_airport.id != flight.takeoff_airport_id:
             await handle_airport_changed(
                 db,

src/graphql_schema/entities/helpers/flight.py

@@ -1,11 +1,16 @@
 import asyncio
 from datetime import datetime
 from typing import List, Type, Literal, Optional, Tuple
+
+from aiohttp import ClientResponseError
 from sqlalchemy import select, delete
 from sqlalchemy.ext.asyncio import AsyncSession
+from starlette.background import BackgroundTasks
 from strawberry.file_uploads import Upload
 
 from database import models
+from external.elevation import ElevationAPI
+from external.gpx_parser import GPXParser
 from external.weather import Weather
 from graphql_schema.types import ComboboxInput
 from upload_utils import delete_file, file_exists, handle_file_upload
@@ -111,11 +116,33 @@ async def handle_airport_changed(
     setattr(flight, f"{type_}_datetime", input_datetime)
 
 
+async def add_terrain_elevation(db: AsyncSession, flight: models.Flight, gpx_filename: str):
+    path = "/app/uploads/tracks"  # TODO vytahnout do configu
+
+    elevation_api = ElevationAPI()
+    gpx_parser = GPXParser(f"{path}/{gpx_filename}")
+
+    coordinates = await gpx_parser.get_coordinates()
+    print("AAAAAAAAAAAAAAAAAAAAAAAAA", coordinates)
+
+    try:
+        elevation = await elevation_api.get_elevation_for_points(coordinates)
+        print("ELEVATION", elevation)
+        tree_with_elevation = gpx_parser.add_terrain_elevation(elevation)
+        output_name = f"terrain_{gpx_filename}"
+        gpx_parser.write(tree_with_elevation, f"{path}/{output_name}")
+        await models.Flight.update(db, {"gpx_track_filename": output_name}, obj=flight)
+
+    except ClientResponseError:
+        print("NEumim elevation!")
+
+
+
 async def handle_upload_gpx(flight: models.Flight, gpx_track: Upload):
     path = "/app/uploads/tracks"
 
-    if flight.gpx_track_filename and file_exists(path + "/" + flight.gpx_track_filename):
-        delete_file(path + "/" + flight.gpx_track_filename)
+    if flight.gpx_track_filename:
+        delete_file(path + "/" + flight.gpx_track_filename, silent=True)
 
     return await handle_file_upload(gpx_track, path)