diff --git a/src/background_jobs/elevation.py b/src/background_jobs/elevation.py index 438b475..cb146dd 100644 --- a/src/background_jobs/elevation.py +++ b/src/background_jobs/elevation.py @@ -1,30 +1,25 @@ from aiohttp import ClientResponseError +from sqlalchemy import select from database import models from database.transaction import get_session from decorators.retryable import retryable -from external.elevation import elevation_api -from external.gpx_parser import GPXParser -from paths import FLIGHT_GPX_TRACK_PATH +from external.elevation import elevation_api, update_track_points_elevation @retryable -async def add_terrain_elevation_to_flight(flight_id: int, gpx_filename: str): - gpx_parser = GPXParser(f"{FLIGHT_GPX_TRACK_PATH}/{gpx_filename}") - coordinates = await gpx_parser.get_coordinates_all() - - try: - elevation = await elevation_api.get_elevation_for_points(coordinates) - tree_with_elevation = gpx_parser.add_terrain_elevation(elevation) - output_name = f"terrain_{gpx_filename}" - gpx_parser.write(tree_with_elevation, f"{FLIGHT_GPX_TRACK_PATH}/{output_name}") - - async with get_session() as db: - await models.Flight.update( - db, {"gpx_track_filename": output_name, "has_terrain_elevation": True}, - id=flight_id - ) - except ClientResponseError as e: - print(e) +async def add_terrain_elevation_to_flight(flight_id: int): + async with get_session() as db: + track_points = ( + await db.scalars( + select(models.TrackPoint) + .join(models.TrackPoint.track) + .join(models.Track.flight) + .filter(models.Flight.id == flight_id)) + ).all() + try: + await update_track_points_elevation(db, track_points) + except ClientResponseError as e: + print(e) @retryable diff --git a/src/database/models.py b/src/database/models.py index 298b9a9..5414773 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -133,6 +133,7 @@ class Track(BaseModel): created_by_id: Mapped[int] = mapped_column(Integer, ForeignKey('user.id')) created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now()) + flight: Mapped['Flight'] = relationship() # created_by: Mapped['User'] = relationship() @@ -149,6 +150,8 @@ class TrackPoint(BaseModel): altitude: Mapped[float] = mapped_column(Float, nullable=True) magnetic_variation: Mapped[float] = mapped_column(Float, nullable=True) + track: Mapped['Track'] = relationship() + class FlightPlanMarker(BaseModel): __tablename__ = "flight_plan_marker" @@ -427,7 +430,7 @@ class Flight(BaseModel): photos: Mapped[List['Photo']] = relationship(foreign_keys=[Photo.flight_id]) created_by: Mapped['User'] = relationship() title_photo: Mapped['Photo'] = relationship(foreign_keys=[title_photo_id]) - + track: Mapped['Track'] = relationship() class Copilot(BaseModel): __tablename__ = "copilot" diff --git a/src/external/elevation.py b/src/external/elevation.py index 6217752..a0e5bc8 100644 --- a/src/external/elevation.py +++ b/src/external/elevation.py @@ -1,13 +1,15 @@ -import asyncio import dataclasses -from typing import List, Dict +from typing import Optional import aiohttp + +from database import models from decorators.retryable import retryable from utils.list_utils import chunks @dataclasses.dataclass class ElevationInfo: + id: Optional[int] lat: float lng: float elevation: float @@ -16,11 +18,11 @@ class ElevationInfo: class ElevationAPI: ELEVATION_ENDPOINT: str - def get_request(self, session: aiohttp.ClientSession, points: List[Dict[str, float]]): + def get_request(self, session: aiohttp.ClientSession, points: list[dict[str, float]]): ... @retryable - async def call_api(self, points: List[Dict[str, float]]): + async def call_api(self, points: list[dict[str, float]]): async with aiohttp.ClientSession() as session: async with self.get_request(session, points) as resp: resp.raise_for_status() @@ -30,44 +32,44 @@ class ElevationAPI: class OpenElevationAPI(ElevationAPI): ELEVATION_ENDPOINT = "https://api.open-elevation.com/api/v1/lookup" - def get_request(self, session: aiohttp.ClientSession, points: List[Dict[str, float]]): + def get_request(self, session: aiohttp.ClientSession, points: list[dict[str, float]]): data = {"locations": [{"latitude": point['lat'], "longitude": point['lng']} for point in points]} return session.post(self.ELEVATION_ENDPOINT, json=data) - async def get_elevation_for_points(self, points: List[Dict[str, float]]) -> List[ElevationInfo]: + async def get_elevation_for_points(self, points: list[dict[str, float]]) -> list[ElevationInfo]: response = await self.call_api(points) - return [ - ElevationInfo(lat=loc['latitude'], lng=loc['longitude'], elevation=loc['elevation']) - for loc in response['results'] - ] + elevation = [] + for i, point in enumerate(points): + response_point = response['results'][i] + elevation.append(ElevationInfo(**point, elevation=response_point['elevation'])) + return elevation class OpenMeteoElevationAPI(ElevationAPI): ELEVATION_ENDPOINT = "https://api.open-meteo.com/v1/elevation" - def get_request(self, session: aiohttp.ClientSession, points: List[Dict[str, float]]): + def get_request(self, session: aiohttp.ClientSession, points: list[dict[str, float]]): latitudes = ",".join(str(point['lat']) for point in points) longitudes = ",".join(str(point['lng']) for point in points) url = f"{self.ELEVATION_ENDPOINT}?latitude={latitudes}&longitude={longitudes}" return session.get(url) - async def get_elevation_for_points(self, points: List[Dict[str, float]]) -> List[ElevationInfo]: + async def get_elevation_for_points(self, points: list[dict[str, float]]) -> list[ElevationInfo]: elevation = [] for chunk in chunks(points, n=100): response = await self.call_api(chunk) - - chunk_elevation = [ - ElevationInfo(**point, elevation=response['elevation'][i]) for i, point in enumerate(points) - ] + chunk_elevation = [] + for i, point in enumerate(chunk): + chunk_elevation.append(ElevationInfo(**point, elevation=response['elevation'][i])) elevation += chunk_elevation return elevation class CombinedElevationAPI: - async def get_elevation_for_points(self, points: List[Dict[str, float]]) -> List[ElevationInfo]: + async def get_elevation_for_points(self, points: list[dict[str, float]]) -> list[ElevationInfo]: for api in (OpenMeteoElevationAPI(), OpenElevationAPI()): elevation = await api.get_elevation_for_points(points) @@ -75,4 +77,11 @@ class CombinedElevationAPI: return elevation +async def update_track_points_elevation(db, track_points: list[models.TrackPoint]): + points = [{"id": p.id, "lat": p.gps_latitude, "lng": p.gps_longitude} for p in track_points] + elevation = await elevation_api.get_elevation_for_points(points) + for point in elevation: + await models.TrackPoint.update(db, {"terrain_elevation": point.elevation}, id=point.id) + + elevation_api = CombinedElevationAPI() diff --git a/src/external/gpx_parser.py b/src/external/gpx_parser.py index c025bf5..6cb89fd 100644 --- a/src/external/gpx_parser.py +++ b/src/external/gpx_parser.py @@ -3,7 +3,6 @@ from datetime import datetime, timedelta from typing import List, Dict, Any from aiocache import cached from lxml import etree -from lxml.etree import _ElementTree class GPXParser: @@ -136,30 +135,3 @@ class GPXParser: async def get_avg_altitude(self): altitudes = await self.get_altitude() return round(sum(altitudes) / len(altitudes), 2) - - def add_terrain_elevation(self, points_with_elevation: List[Dict[str, float]]): - track_points = self.run_xpath("//gpx:trkpt") - # TODO: open elevation API umi jen presnost na 6 desetinnych mist! - track_points_index = {(float(n.attrib['lat']), float(n.attrib['lon'])): n for n in track_points} - - for point_with_elevation in points_with_elevation: - lat = point_with_elevation['lat'] - lng = point_with_elevation['lng'] - elevation = point_with_elevation['elevation'] - target_node = track_points_index.get((lat, lng)) - - if target_node is None: - continue - - extensions = target_node.find("./extensions", self.gpx.getroot().nsmap) - if extensions is not None: - extensions.append(etree.XML(f"{elevation}")) - else: - target_node.append( - etree.XML(f"{elevation}") - ) - - return self.gpx - - def write(self, tree: _ElementTree, output: str): - tree.write(output) diff --git a/src/scripts/elevation.py b/src/scripts/elevation.py index b641d69..8009366 100644 --- a/src/scripts/elevation.py +++ b/src/scripts/elevation.py @@ -1,11 +1,13 @@ import asyncio import sys from sqlalchemy import select + sys.path.insert(0, "/app/src") from paths import FLIGHT_GPX_TRACK_PATH # noqa from database import async_session, models # noqa -from external.elevation import elevation_api # noqa +from external.elevation import elevation_api, update_track_points_elevation # noqa from external.gpx_parser import GPXParser # noqa +from database.transaction import get_session async def add_elevation_to_photos(): @@ -32,44 +34,23 @@ async def add_elevation_to_photos(): async def add_elevation_to_tracks(): - async with async_session() as session: - flights = (await session.scalars( - select(models.Flight) - .filter(models.Flight.has_terrain_elevation.is_(False)) - .filter(models.Flight.gpx_track_filename.isnot(None)) + async with get_session() as db: + track_points = (await db.session.scalars( + select(models.TrackPoint) + .filter(models.TrackPoint.terrain_elevation.is_(None)) )).all() - if not flights: + if not track_points: print("all done") return - - for flight in flights: - gpx_file = f"{FLIGHT_GPX_TRACK_PATH}/{flight.gpx_track_filename}" - try: - gpx = GPXParser(gpx_file) - except OSError: - print(f"ERROR, cannot open {gpx_file=}, skipping") - continue - - coordinates = await gpx.get_coordinates_all() - elevation = await elevation_api.get_elevation_for_points(coordinates) - - gpx_with_elevation = gpx.add_terrain_elevation(elevation) - - output_name = f"terrain_{flight.gpx_track_filename[-30:]}" - gpx.write(gpx_with_elevation, output=f"{FLIGHT_GPX_TRACK_PATH}/{output_name}") - await models.Flight.update( - db_session=session, obj=flight, data={ - "has_terrain_elevation": True, - "gpx_track_filename": output_name - } - ) - await session.flush() - await session.commit() + await update_track_points_elevation(db, track_points) async def run_all(): - await asyncio.gather(add_elevation_to_photos(), add_elevation_to_tracks()) + await asyncio.gather( + add_elevation_to_photos(), + add_elevation_to_tracks() + ) if __name__ == "__main__":