From ed5df4f9535b8bf09b8effe8026ef142b79734f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Kv=C3=A1=C4=8Dek?= Date: Mon, 4 Sep 2023 15:34:08 +0200 Subject: [PATCH] Parsovani GPX --- .env.dist | 1 + docker-compose.prod.yml | 2 - docker-compose.yml | 2 + src/database/config.py | 2 +- src/external/elevation.py | 25 +++++ src/external/gpx_parser.py | 93 +++++++++++++++++++ src/gpx_test.py | 13 +++ src/graphql_schema/entities/flight.py | 47 ++++++---- src/graphql_schema/entities/helpers/flight.py | 31 ++++++- 9 files changed, 194 insertions(+), 22 deletions(-) create mode 100644 src/external/elevation.py create mode 100644 src/external/gpx_parser.py create mode 100644 src/gpx_test.py diff --git a/.env.dist b/.env.dist index 39e5c13..bcccc38 100644 --- a/.env.dist +++ b/.env.dist @@ -6,3 +6,4 @@ MYSQL_HOST: "127.0.0.1" MYSQL_DATABASE: "poletani_cz" MYSQL_USER: "mysql_user" MYSQL_PASSWORD: "****" +APP_ENV: "production" diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 9662e61..86d4e6d 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,7 +1,5 @@ version: "3.3" -# testovaci koment - services: api: image: "docker.kvacek.cz/poletani/api:latest" diff --git a/docker-compose.yml b/docker-compose.yml index 53da441..415400a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,8 +20,10 @@ services: environment: TZ: "Europe/Prague" API_URL: "http://localhost:8000" + APP_ENV: "development" ALLOW_CORS_ORIGINS: "http://localhost:9000 http://localhost:9001" + scrapy: build: context: ../crawlers diff --git a/src/database/config.py b/src/database/config.py index 58ca878..e1d5e80 100644 --- a/src/database/config.py +++ b/src/database/config.py @@ -14,7 +14,7 @@ def get_database_url(): def create_db_engine(): database_url = get_database_url() - return create_async_engine(database_url, future=True, echo=True) + return create_async_engine(database_url, future=True, echo=True, pool_pre_ping=True) engine = create_db_engine() diff --git a/src/external/elevation.py b/src/external/elevation.py new file mode 100644 index 0000000..195737e --- /dev/null +++ b/src/external/elevation.py @@ -0,0 +1,25 @@ +from typing import List, Tuple, Dict + +import aiohttp + + +class ElevationAPI: + ELEVATION_ENDPOINT = "https://api.open-elevation.com/api/v1/lookup" + + def get_request(self, points: List[Dict[str, float]]): + return {"locations": [{"latitude": point['lat'], "longitude": point['lng']} for point in points]} + + async def call_api(self, points: List[Dict[str, float]]): + async with aiohttp.ClientSession() as session: + async with session.post(self.ELEVATION_ENDPOINT, json=self.get_request(points)) as resp: + resp.raise_for_status() + return await resp.json() + + async def get_elevation_for_points(self, points: List[Dict[str, float]]) -> List[dict]: + response = await self.call_api(points) + + return [{ + "lat": loc['latitude'], + "lng": loc['longitude'], + "elevation": loc['elevation'] + } for loc in response['results']] \ No newline at end of file diff --git a/src/external/gpx_parser.py b/src/external/gpx_parser.py new file mode 100644 index 0000000..f331782 --- /dev/null +++ b/src/external/gpx_parser.py @@ -0,0 +1,93 @@ +from datetime import datetime +from typing import List, Dict + +from aiocache import cached +from lxml import etree +from lxml.etree import _ElementTree + + +class GPXParser: + def __init__(self, file: str): + self.file = file + self.gpx = etree.parse(self.file) + self.namespace = None + self.precision_digits = 6 + self.set_namespace() + + def set_namespace(self): + namespace = self.gpx.getroot().nsmap.get(None) + self.namespace = {'gpx': namespace} + + def run_xpath(self, path: str): + return self.gpx.xpath(path, namespaces=self.namespace) + + @cached() + async def get_times(self): + nodes = self.run_xpath("//gpx:time") + return [datetime.fromisoformat(node.text).astimezone() for node in nodes] + + @cached() + async def get_coordinates(self) -> List[Dict[str, float]]: + nodes = self.run_xpath("//gpx:trkpt") + return [{"lat": float(node.attrib["lat"]), "lng": float(node.attrib['lon'])} for node in nodes] + + @cached() + async def get_speed(self) -> List[int]: + nodes = self.run_xpath("//gpx:speed") + return [int(node.text) for node in nodes] + + @cached() + async def get_magnetic_variation(self) -> List[int]: + nodes = self.run_xpath("//gpx:magvar") + return [int(node.text) for node in nodes] + + @cached() + async def get_altitude(self) -> List[float]: + nodes = self.run_xpath("//gpx:ele") + return [float(node.text) for node in nodes] + + @cached() + async def get_terrain_elevation(self) -> List[float]: + nodes = self.run_xpath("//gpx:terrain_elevation") + return [float(node.text) for node in nodes] + + @cached() + async def get_max_speed(self): + return max(await self.get_speed()) + + @cached() + async def get_avg_speed(self): + speeds = await self.get_speed() + return round(sum(speeds) / len(speeds), 2) + + @cached() + async def get_max_altitude(self): + return max(await self.get_altitude()) + + @cached() + async def get_avg_altitude(self): + altitudes = await self.get_altitude() + return round(sum(altitudes) / len(altitudes), 2) + + def add_terrain_elevation(self, points_with_elevation: List[Dict[str, float]]): + track_points = self.run_xpath("//gpx:trkpt") + # TODO: open elevation API umi jen presnost na 6 desetinnych mist! + track_points_index = {(float(n.attrib['lat']), float(n.attrib['lon'])): n for n in track_points} + + for point_with_elevation in points_with_elevation: + lat = point_with_elevation['lat'] + lng = point_with_elevation['lng'] + elevation = point_with_elevation['elevation'] + target_node = track_points_index.get((lat, lng)) + + if target_node is None: + continue + + extensions = target_node.find("./extensions", self.gpx.getroot().nsmap) + extensions.append(etree.XML(f"{elevation}")) + + return self.gpx + + def write(self, tree: _ElementTree, output: str): + print(f"ZAPISUJI DO {output}") + tree.write(output) diff --git a/src/gpx_test.py b/src/gpx_test.py new file mode 100644 index 0000000..3c6917d --- /dev/null +++ b/src/gpx_test.py @@ -0,0 +1,13 @@ +import asyncio +from external.elevation import ElevationAPI +from external.gpx_parser import GPXParser + + +async def test(): + elevation_api = ElevationAPI() + parser = GPXParser("./uploads/tracks/37b979cb-71c0-4d09-a2c1-cfdad5f7a0cf-OK-AUR_28_AUR_Bristell_NG5_Zapisnik_letu_2023-08-04-00 00_2023-08-04-12 00(1).gpx") + points = await parser.get_coordinates() + elevation = await elevation_api.get_elevation_for_points(points) + parser.add_terrain_elevation(elevation) + +asyncio.run(test()) \ No newline at end of file diff --git a/src/graphql_schema/entities/flight.py b/src/graphql_schema/entities/flight.py index 709ba46..1325f5d 100644 --- a/src/graphql_schema/entities/flight.py +++ b/src/graphql_schema/entities/flight.py @@ -1,5 +1,5 @@ import asyncio -from datetime import timedelta +from datetime import timedelta, datetime from typing import List, Optional, Annotated, TYPE_CHECKING, Tuple import strawberry from fastapi import HTTPException @@ -11,6 +11,7 @@ from database import models from database.models import flight_has_copilot from decorators.endpoints import authenticated_user_only from decorators.error_logging import error_logging +from external.gpx_parser import GPXParser from graphql_schema.dataloaders.aircraft import aircraft_dataloader from graphql_schema.dataloaders.airport import airport_dataloader from graphql_schema.dataloaders.copilots import flight_copilots_dataloader @@ -23,7 +24,7 @@ from graphql_schema.entities.photo import Photo from graphql_schema.entities.poi import PointOfInterest from graphql_schema.sqlalchemy_to_strawberry_type import strawberry_sqlalchemy_type, strawberry_sqlalchemy_input from upload_utils import get_public_url -from .helpers.flight import handle_aircraft_save, handle_track_edit, handle_copilots_edit, handle_weather_info, get_airports, handle_upload_gpx, handle_airport_changed +from .helpers.flight import handle_aircraft_save, handle_track_edit, handle_copilots_edit, handle_weather_info, get_airports, handle_upload_gpx, handle_airport_changed, add_terrain_elevation from ..types import ComboboxInput if TYPE_CHECKING: @@ -51,10 +52,16 @@ class Point: @strawberry.type class GPXTrack: - points: List[Point] + coordinates: List[Point] speed: List[float] - elevation: List[float] + altitude: List[float] + magnetic_variation: List[float] terrain_elevation: List[float] + time: List[datetime] + max_speed: float + avg_speed: float + max_altitude: float + avg_altitude: float @strawberry_sqlalchemy_type(models.Flight) @@ -97,18 +104,22 @@ class Flight: if not root.gpx_track_filename: return None - path = "/app/uploads/tracks" + try: + gpx_parser = GPXParser(f"/app/uploads/tracks/{root.gpx_track_filename}") + except OSError: + return None - tree = etree.parse(f"{path}/{root.gpx_track_filename}") - points = tree.findall("//trk") - speed = tree.xpath("//extensions/speed") - elevation = tree.xpath('//ele') - print(tree.getroot().find("trk"), list(points), speed, elevation) return GPXTrack( - points=[Point(lat=50.123, lng=14.324243)], - speed=[], - elevation=[], - terrain_elevation=[] + coordinates=[Point(**point) for point in await gpx_parser.get_coordinates()], + speed=await gpx_parser.get_speed(), + altitude=await gpx_parser.get_altitude(), + terrain_elevation=await gpx_parser.get_terrain_elevation(), + time=await gpx_parser.get_times(), + max_speed=await gpx_parser.get_max_speed(), + avg_speed=await gpx_parser.get_avg_speed(), + max_altitude=await gpx_parser.get_max_altitude(), + avg_altitude=await gpx_parser.get_avg_altitude(), + magnetic_variation=await gpx_parser.get_magnetic_variation(), ) def load_gpx_track_url(root): @@ -131,7 +142,7 @@ class Flight: takeoff_weather_info: Optional[WeatherInfo] = strawberry.field(resolver=load_takeoff_weather_info) landing_weather_info: Optional[WeatherInfo] = strawberry.field(resolver=load_landing_weather_info) photos: List[Photo] = strawberry.field(resolver=load_photos) - gpx_track_url: Optional[str] = strawberry.field(resolver=load_gpx_track_url) + gpx_track_url: Optional[str] = strawberry.field(resolver=load_gpx_track_url) # TODO: odstranit gpx_track: Optional[GPXTrack] = strawberry.field(resolver=load_gpx_track) @@ -222,8 +233,8 @@ class CreateFlightMutation: @strawberry.type class EditFlightMutation: @strawberry_sqlalchemy_input(models.Flight, exclude_fields=[ - "id", "aircraft_id", "deleted", "landing_airport_id", "takeoff_airport_id" - "takeoff_weather_info_id", "landing_weather_info_id", "gpx_track_filename" + "id", "aircraft_id", "deleted", "landing_airport_id", "takeoff_airport_id", + "takeoff_weather_info_id", "landing_weather_info_id", "gpx_track_filename" ], all_optional=True) class EditFlightInput: gpx_track: Optional[Upload] = None # TODO: poresit validaci uploadovaneho souboru! @@ -253,7 +264,9 @@ class EditFlightMutation: if input.gpx_track is not None: data['gpx_track_filename'] = await handle_upload_gpx(flight, input.gpx_track) + info.context.background_tasks.add_task(add_terrain_elevation, flight=flight, gpx_filename=data['gpx_track_filename'], db=db) + # TODO: nasledujici metody volat i pokud se zmenil cas vzletu! if input.takeoff_airport and input.takeoff_airport.id != flight.takeoff_airport_id: await handle_airport_changed( db, diff --git a/src/graphql_schema/entities/helpers/flight.py b/src/graphql_schema/entities/helpers/flight.py index 7928972..b7f255a 100644 --- a/src/graphql_schema/entities/helpers/flight.py +++ b/src/graphql_schema/entities/helpers/flight.py @@ -1,11 +1,16 @@ import asyncio from datetime import datetime from typing import List, Type, Literal, Optional, Tuple + +from aiohttp import ClientResponseError from sqlalchemy import select, delete from sqlalchemy.ext.asyncio import AsyncSession +from starlette.background import BackgroundTasks from strawberry.file_uploads import Upload from database import models +from external.elevation import ElevationAPI +from external.gpx_parser import GPXParser from external.weather import Weather from graphql_schema.types import ComboboxInput from upload_utils import delete_file, file_exists, handle_file_upload @@ -111,11 +116,33 @@ async def handle_airport_changed( setattr(flight, f"{type_}_datetime", input_datetime) +async def add_terrain_elevation(db: AsyncSession, flight: models.Flight, gpx_filename: str): + path = "/app/uploads/tracks" # TODO vytahnout do configu + + elevation_api = ElevationAPI() + gpx_parser = GPXParser(f"{path}/{gpx_filename}") + + coordinates = await gpx_parser.get_coordinates() + print("AAAAAAAAAAAAAAAAAAAAAAAAA", coordinates) + + try: + elevation = await elevation_api.get_elevation_for_points(coordinates) + print("ELEVATION", elevation) + tree_with_elevation = gpx_parser.add_terrain_elevation(elevation) + output_name = f"terrain_{gpx_filename}" + gpx_parser.write(tree_with_elevation, f"{path}/{output_name}") + await models.Flight.update(db, {"gpx_track_filename": output_name}, obj=flight) + + except ClientResponseError: + print("NEumim elevation!") + + + async def handle_upload_gpx(flight: models.Flight, gpx_track: Upload): path = "/app/uploads/tracks" - if flight.gpx_track_filename and file_exists(path + "/" + flight.gpx_track_filename): - delete_file(path + "/" + flight.gpx_track_filename) + if flight.gpx_track_filename: + delete_file(path + "/" + flight.gpx_track_filename, silent=True) return await handle_file_upload(gpx_track, path)