From ca3f6ef07ca90d112d48532203ea65da833e131f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Kv=C3=A1=C4=8Dek?= Date: Mon, 30 Sep 2024 06:43:11 +0200 Subject: [PATCH] Lepsi stahovani informaci o pocasi --- src/background_jobs/weather.py | 25 ++-- src/database/models.py | 2 + .../dataloaders/single_model.py | 2 +- src/graphql_schema/entities/aircraft.py | 4 +- src/graphql_schema/entities/airport.py | 1 - src/graphql_schema/entities/flight_plan.py | 2 +- .../entities/resolvers/aircraft.py | 6 +- src/graphql_schema/entities/resolvers/base.py | 6 +- .../entities/resolvers/flight.py | 2 +- .../entities/resolvers/flight_plan.py | 128 ++++++++++-------- src/graphql_schema/entities/types/types.py | 10 +- src/scripts/download_weather.py | 68 ++++++++-- src/utils/flight_plan_helpers.py | 31 +++++ src/utils/list_utils.py | 7 + 14 files changed, 198 insertions(+), 96 deletions(-) create mode 100644 src/utils/flight_plan_helpers.py diff --git a/src/background_jobs/weather.py b/src/background_jobs/weather.py index 02e6126..bcb33f5 100644 --- a/src/background_jobs/weather.py +++ b/src/background_jobs/weather.py @@ -14,7 +14,7 @@ async def get_weather(date_time: datetime, gps: tuple[float, float]): return None return { - "datetime": weather['datetime'], + "datetime": date_time, # nemuzu pouzit weather['datetime'], protoze to mi vraci jen cele hodiny "qnh": weather['pressure_msl'], "temperature_surface": weather['temperature_2m'], "dewpoint_surface": weather['dewpoint_2m'], @@ -44,18 +44,27 @@ async def download_weather_for_flight(date_time: datetime, flight_id: int, airpo await models.Flight.update(db, obj=flight, data={f"{type_}_weather_info_id": weather_model.id}) -async def download_weather_for_plan(date_time: datetime, flight_plan_id: int, gps: tuple[float, float]): +async def download_weather_for_plan_marker(date_time: datetime, marker_id: int): + async with get_session() as db: + marker = await models.FlightPlanMarker.get_one(db, id=marker_id) + + if not marker.gps_latitude or not marker.gps_longitude: + return + + gps = (marker.gps_latitude, marker.gps_longitude) + marker_weather_info_id = marker.weather_info_id + data = await get_weather(date_time, gps) + if not data: + return async with get_session() as db: - plan = await models.FlightPlan.get_one(db, id=flight_plan_id) - - if plan.weather_info_id: - await models.WeatherInfo.update(db, id=plan.weather_info.id, data=data) + if marker_weather_info_id: + await models.WeatherInfo.update(db, id=marker_weather_info_id, data=data) else: weather_model = await models.WeatherInfo.create(db, data=data) - await models.FlightPlan.update( + await models.FlightPlanMarker.update( db, - obj=plan, + id=marker_id, data={"weather_info_id": weather_model.id} ) diff --git a/src/database/models.py b/src/database/models.py index 5414773..c6a65f9 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -168,6 +168,8 @@ class FlightPlanMarker(BaseModel): gps_longitude: Mapped[float] = mapped_column(Float, nullable=False) + + class Airport(BaseModel): __tablename__ = "airport" diff --git a/src/graphql_schema/dataloaders/single_model.py b/src/graphql_schema/dataloaders/single_model.py index 68dba5f..cfba301 100644 --- a/src/graphql_schema/dataloaders/single_model.py +++ b/src/graphql_schema/dataloaders/single_model.py @@ -14,7 +14,7 @@ airport_dataloader = create_dataloader(models.Airport) aircraft_dataloader = create_dataloader(models.Aircraft) event_dataloader = create_dataloader(models.Event) organizations_dataloader = create_dataloader(models.Organization) -airport_weather_info_loader = create_dataloader(models.WeatherInfo) +weather_info_loader = create_dataloader(models.WeatherInfo) track_dataloader = create_dataloader(models.Track) poi_dataloader = create_dataloader(models.PointOfInterest) poi_type_dataloader = create_dataloader(models.PointOfInterestType) diff --git a/src/graphql_schema/entities/aircraft.py b/src/graphql_schema/entities/aircraft.py index da859be..d4dc464 100644 --- a/src/graphql_schema/entities/aircraft.py +++ b/src/graphql_schema/entities/aircraft.py @@ -62,9 +62,9 @@ class AircraftMutation: @error_logging @authenticated_user_only() async def edit_aircraft(root, info, id: int, input: EditAircraftInput) -> Aircraft: - return await AircraftMutationResolver().update(id, user_id=info.context.user_id, data=input) + return await AircraftMutationResolver().update(id, info.context, data=input) @strawberry.mutation @authenticated_user_only() async def delete_aircraft(self, info, id: int) -> Aircraft: - return await AircraftMutationResolver().delete(info.context.user_id, id) + return await AircraftMutationResolver().delete(info.context, id) diff --git a/src/graphql_schema/entities/airport.py b/src/graphql_schema/entities/airport.py index 6221b0c..1637aa3 100644 --- a/src/graphql_schema/entities/airport.py +++ b/src/graphql_schema/entities/airport.py @@ -11,7 +11,6 @@ from graphql_schema.entities.types.types import Airport class AirportQueries: @strawberry.field() @error_logging - # @authenticated_user_only() async def airports(root, info) -> List[Airport]: return await BaseQueryResolver(Airport, models.Airport).get_list(info.context.user_id) diff --git a/src/graphql_schema/entities/flight_plan.py b/src/graphql_schema/entities/flight_plan.py index a720def..932875e 100644 --- a/src/graphql_schema/entities/flight_plan.py +++ b/src/graphql_schema/entities/flight_plan.py @@ -50,4 +50,4 @@ class FlightPlanMutation: @error_logging @authenticated_user_only() async def edit_flight_plan(root, info, id: int, input: EditFlightPlanInput) -> FlightPlan: - return await FlightPlanMutationResolver().update(id, input, info.context.user_id) + return await FlightPlanMutationResolver().update(info.context, id, input) diff --git a/src/graphql_schema/entities/resolvers/aircraft.py b/src/graphql_schema/entities/resolvers/aircraft.py index 0192c86..e74a693 100644 --- a/src/graphql_schema/entities/resolvers/aircraft.py +++ b/src/graphql_schema/entities/resolvers/aircraft.py @@ -1,8 +1,6 @@ from operator import or_ from typing import Set, Optional - from sqlalchemy import and_ - from database import models from database.transaction import get_session from graphql_schema.entities.helpers.combobox import handle_combobox_save @@ -77,7 +75,7 @@ class AircraftMutationResolver(BaseMutationResolver): return await self._do_create(db, data=input_data) - async def update(self, id: int, user_id: int, data: EditAircraftInput) -> Aircraft: + async def update(self, context, id: int, data: EditAircraftInput) -> Aircraft: update_data = data.to_dict() async with get_session() as db: if data.organization: @@ -85,7 +83,7 @@ class AircraftMutationResolver(BaseMutationResolver): db, models.Organization, input=data.organization, - user_id=user_id, + user_id=context.user_id, ) return await self._do_update(db, id, update_data) diff --git a/src/graphql_schema/entities/resolvers/base.py b/src/graphql_schema/entities/resolvers/base.py index 37f02f1..25549c4 100644 --- a/src/graphql_schema/entities/resolvers/base.py +++ b/src/graphql_schema/entities/resolvers/base.py @@ -116,14 +116,14 @@ class BaseMutationResolver(BaseResolver): async with get_session() as db: return await self._do_create(db, input_data) - async def update(self, id: int, data: BaseGraphqlInputType, user_id: int) -> GQL_TYPE: + async def update(self, context, id: int, data: BaseGraphqlInputType, user_id: int) -> GQL_TYPE: async with get_session() as db: item = await self._get_one(db, id, user_id) return await self._do_update(db, item, data.to_dict()) - async def delete(self, user_id: int, id: int, **kwargs) -> GQL_TYPE: + async def delete(self, context, id: int, **kwargs) -> GQL_TYPE: async with get_session() as db: - model = await self._get_one(db, id, user_id) + model = await self._get_one(db, id, context.user_id) if hasattr(self.model, "deleted"): model = await self.model.update(db, obj=model, data=dict(deleted=True)) diff --git a/src/graphql_schema/entities/resolvers/flight.py b/src/graphql_schema/entities/resolvers/flight.py index d64a2a5..88c618f 100644 --- a/src/graphql_schema/entities/resolvers/flight.py +++ b/src/graphql_schema/entities/resolvers/flight.py @@ -109,7 +109,7 @@ class FlightMutationResolver(BaseMutationResolver): if input.track is not None: await handle_track_edit(db=db, flight_id=flight.id, track=input.track, user_id=user_id) - if data['gpx_track_filename']: + if 'gpx_track_filename' in data: await save_track_from_gpx_to_db(gpx_filename=data['gpx_track_filename'], flight_id=flight.id) schedule_background_tasks(flight.id, data, context) diff --git a/src/graphql_schema/entities/resolvers/flight_plan.py b/src/graphql_schema/entities/resolvers/flight_plan.py index 4b69c67..b20c8f6 100644 --- a/src/graphql_schema/entities/resolvers/flight_plan.py +++ b/src/graphql_schema/entities/resolvers/flight_plan.py @@ -10,6 +10,8 @@ from graphql_schema.entities.resolvers.base import BaseMutationResolver, BaseQue from graphql_schema.entities.resolvers.flight import handle_aircraft_save from graphql_schema.entities.types.mutation_input import CreateFlightPlanInput, EditFlightPlanInput from graphql_schema.entities.types.types import FlightPlan +from utils.flight_plan_helpers import refresh_markers_weather_info +from utils.list_utils import evenly_spaced_elements from utils.str_utils import random_str @@ -47,61 +49,24 @@ class FlightPlanMutationResolver(BaseMutationResolver): def __init__(self): super().__init__(graphql_type=FlightPlan, model=models.FlightPlan) - @staticmethod - async def save_markers(db, flight_plan: models.FlightPlan, markers: list): - position = 0 - - for marker in markers: - - if marker.type == 'poi': - assert bool(marker.point_of_interest_id) - - if marker.type == 'airport': - assert bool(marker.airport_id) - - await models.FlightPlanMarker.create(db, data={ - "position": position, - "flight_plan_id": flight_plan.id, - "airport_id": marker.airport_id, - "point_of_interest_id": marker.point_of_interest_id, - "type": marker.type, - "name": marker.name, - "gps_latitude": marker.gps_latitude, - "gps_longitude": marker.gps_longitude - }) - position += 1 - - @staticmethod - async def reset_plan_markers(db, flight_plan: models.FlightPlan): - await db.execute( - delete(models.FlightPlanMarker) - .filter(models.FlightPlanMarker.flight_plan_id == flight_plan.id) - ) - async def create(self, context, data: CreateFlightPlanInput) -> FlightPlan: input_data = data.to_dict() input_data['created_by_id'] = context.user_id async with get_session() as db: flight_plan = await self._do_create(db, data=input_data) - await self.save_markers(db, flight_plan, data.markers) + markers = await self.save_markers(db, flight_plan, data.markers) + await refresh_markers_weather_info( + planned_takeoff_datetime=flight_plan.planned_takeoff_datetime, + planned_speed=flight_plan.planned_speed, + markers=markers, + background_tasks=context.background_tasks + ) return flight_plan - async def save_copilots(self, db, flight_plan_id: int, copilots: list, user_id: int): - await db.execute(delete(flight_plan_has_copilot).filter_by(flight_plan_id=flight_plan_id)) - copilots = await asyncio.gather(*[ - handle_combobox_save(db, models.Copilot, copilot, user_id) for copilot in copilots - ]) - - for copilot_id in copilots: - await db.execute(insert(flight_plan_has_copilot).values( - flight_plan_id=flight_plan_id, - copilot_id=copilot_id, - token=random_str(64) - )) - - async def update(self, id: int, data: EditFlightPlanInput, user_id: int) -> FlightPlan: + async def update(self, context, id: int, data: EditFlightPlanInput) -> FlightPlan: input_data = data.to_dict() + user_id = context.user_id async with get_session() as db: flight_plan_model = await self._get_one(db, id=id, created_by_id=user_id) @@ -109,19 +74,29 @@ class FlightPlanMutationResolver(BaseMutationResolver): if data.aircraft is not None: input_data['aircraft_id'] = await handle_aircraft_save(db, user_id, data.aircraft) + markers = [] if data.markers is not None: await self.reset_plan_markers(db, flight_plan_model) - await self.save_markers(db, flight_plan_model, data.markers) + markers = await self.save_markers(db, flight_plan_model, data.markers) + await refresh_markers_weather_info( + planned_takeoff_datetime=flight_plan_model.planned_takeoff_datetime, + planned_speed=flight_plan_model.planned_speed, + markers=markers, + background_tasks=context.background_tasks + ) if flight_plan_model.is_default_name: - if data.markers: - markers = data.markers - else: - markers = (await db.scalars(select(models.FlightPlanMarker).filter(models.FlightPlanMarker.flight_plan_id == id))).all() + if not markers: + markers = await db.scalars( + select(models.FlightPlanMarker) + .filter(models.FlightPlanMarker.flight_plan_id == id) + ).all() used_markers = evenly_spaced_elements(markers, 5) input_data['name'] = " - ".join(m.name for m in used_markers) + # TODO: pokud se zmenila rychlost nebo datum/cas letu, je potreba stahnout nove pocasi! + if data.copilots is not None: await self.save_copilots( db, @@ -134,8 +109,49 @@ class FlightPlanMutationResolver(BaseMutationResolver): return flight_plan -def evenly_spaced_elements(lst: list, count: int) -> list: - if count > len(lst): - return lst - interval = (len(lst) - 1) / (count - 1) if count > 1 else 0 - return [lst[int(round(i * interval))] for i in range(count)] + @staticmethod + async def save_markers(db, flight_plan: models.FlightPlan, markers: list): + position = 0 + + marker_objects = [] + for marker in markers: + if marker.type == 'poi': + assert bool(marker.point_of_interest_id) + + if marker.type == 'airport': + assert bool(marker.airport_id) + + marker_obj = await models.FlightPlanMarker.create(db, data={ + "position": position, + "flight_plan_id": flight_plan.id, + "airport_id": marker.airport_id, + "point_of_interest_id": marker.point_of_interest_id, + "type": marker.type, + "name": marker.name, + "gps_latitude": marker.gps_latitude, + "gps_longitude": marker.gps_longitude + }) + marker_objects.append(marker_obj) + position += 1 + + return marker_objects + + @staticmethod + async def reset_plan_markers(db, flight_plan: models.FlightPlan): + await db.execute( + delete(models.FlightPlanMarker) + .filter(models.FlightPlanMarker.flight_plan_id == flight_plan.id) + ) + + async def save_copilots(self, db, flight_plan_id: int, copilots: list, user_id: int): + await db.execute(delete(flight_plan_has_copilot).filter_by(flight_plan_id=flight_plan_id)) + copilots = await asyncio.gather(*[ + handle_combobox_save(db, models.Copilot, copilot, user_id) for copilot in copilots + ]) + + for copilot_id in copilots: + await db.execute(insert(flight_plan_has_copilot).values( + flight_plan_id=flight_plan_id, + copilot_id=copilot_id, + token=random_str(64) + )) diff --git a/src/graphql_schema/entities/types/types.py b/src/graphql_schema/entities/types/types.py index 30473fe..a1f8832 100644 --- a/src/graphql_schema/entities/types/types.py +++ b/src/graphql_schema/entities/types/types.py @@ -17,7 +17,7 @@ from graphql_schema.dataloaders.multi_models import ( ) from graphql_schema.dataloaders.single_model import ( poi_dataloader, poi_type_dataloader, event_dataloader, aircraft_dataloader, airport_dataloader, - airport_weather_info_loader, organizations_dataloader, flight_dataloader, photo_adjustment_dataloader, + weather_info_loader, organizations_dataloader, flight_dataloader, photo_adjustment_dataloader, photo_dataloader, user_dataloader, track_dataloader ) from graphql_schema.permissions import IsAuthenticated @@ -162,10 +162,10 @@ class Flight: title_photo: Optional[Photo] = strawberry.field(resolver=lambda root: photo_dataloader.load(root.title_photo_id)) turn_points: List[FlightTurnPoint] = strawberry.field(resolver=lambda root: flight_turn_points_dataloader.load(root.id)) takeoff_weather_info: Optional[WeatherInfo] = strawberry.field( - resolver=lambda root: airport_weather_info_loader.load(root.takeoff_weather_info_id) + resolver=lambda root: weather_info_loader.load(root.takeoff_weather_info_id) ) landing_weather_info: Optional[WeatherInfo] = strawberry.field( - resolver=lambda root: airport_weather_info_loader.load(root.landing_weather_info_id) + resolver=lambda root: weather_info_loader.load(root.landing_weather_info_id) ) photos: List[Photo] = strawberry.field(resolver=lambda root: photos_dataloader.load(root.id)) track: Optional[Track] = strawberry.field(resolver=lambda root: track_dataloader.load(root.track_id)) @@ -181,7 +181,9 @@ class Flight: @strawberry_sqlalchemy_type(model=models.FlightPlanMarker) class FlightPlanMarker: - pass + weather_info: Optional[WeatherInfo] = strawberry.field( + resolver=lambda root: weather_info_loader.load(root.weather_info_id) + ) @strawberry.type diff --git a/src/scripts/download_weather.py b/src/scripts/download_weather.py index bde288d..e5fdafa 100644 --- a/src/scripts/download_weather.py +++ b/src/scripts/download_weather.py @@ -1,22 +1,60 @@ -from sqlalchemy import select +import asyncio +import sys +from collections import defaultdict +from typing import Any +from fastapi import BackgroundTasks +from sqlalchemy import select, Result +from sqlalchemy.ext.asyncio import AsyncSession + +sys.path.insert(0, "/app/src") + +from utils.flight_plan_helpers import refresh_markers_weather_info +from database.models import FlightPlan from database import models from database.transaction import get_session -class FlighPlanWeather: - @staticmethod - async def get_plans() -> list: - async with get_session() as db: - markers_without_weather = (await db.execute( - select(models.FlightPlanMarker).filter(models.FlightPlanMarker.weather_info_id.is_(None)) - )) - flight_plan_ids = {m.flight_plan_id for m in markers_without_weather} +async def get_plans(db: AsyncSession) -> list[Any] | Result[tuple[FlightPlan, Any]]: + markers_without_weather = (await db.scalars( + select(models.FlightPlanMarker).filter(models.FlightPlanMarker.weather_info_id.is_(None)) + )) + flight_plan_ids = {m.flight_plan_id for m in markers_without_weather} + flight_plan_ids.add(11) + if not flight_plan_ids: + return [] - if not flight_plan_ids: - return [] + return (await db.execute( + select(models.FlightPlan, models.FlightPlanMarker) + .join(models.FlightPlan.markers) + .filter(models.FlightPlan.planned_takeoff_datetime.is_not(None)) + .filter(models.FlightPlan.id.in_(flight_plan_ids)) + )) - return (await db.execute( - select(models.FlightPlan, models.FlightPlan.markers) - .filter(models.FlightPlan.id.in_(flight_plan_ids)) - )) +async def download_flight_plan_weather(): + tasks = BackgroundTasks() + + async with get_session() as db: + plans_with_markers_without_weather = await get_plans(db) + + markers_by_plan = defaultdict(list) + for plan, marker in plans_with_markers_without_weather: + markers_by_plan[plan].append(marker) + + for plan, markers in markers_by_plan.items(): + await refresh_markers_weather_info( + plan.planned_takeoff_datetime, + plan.planned_speed, markers, + background_tasks=tasks + ) + + await tasks() + + +async def run_all(): + await asyncio.gather(download_flight_plan_weather()) + + +if __name__ == "__main__": + loop = asyncio.get_event_loop() + loop.run_until_complete(run_all()) diff --git a/src/utils/flight_plan_helpers.py b/src/utils/flight_plan_helpers.py new file mode 100644 index 0000000..cf09270 --- /dev/null +++ b/src/utils/flight_plan_helpers.py @@ -0,0 +1,31 @@ +from datetime import datetime, timedelta +from fastapi import BackgroundTasks +from background_jobs.weather import download_weather_for_plan_marker +from database import models +from utils.gps import get_distance + + +async def refresh_markers_weather_info( + planned_takeoff_datetime: datetime, planned_speed: float, + markers: list[models.FlightPlanMarker], background_tasks: BackgroundTasks +): + if not planned_takeoff_datetime: + return + + marker_date_time = planned_takeoff_datetime + previous_marker = markers[0] if markers else None + for marker in markers: + if marker_date_time: + # TODO: tohle by se melo volat i v pripade zmeny planovaneho casu vzletu i planovane rychlosti + distance_from_prev = get_distance( + a=(previous_marker.gps_latitude, previous_marker.gps_longitude), + b=(marker.gps_latitude, marker.gps_longitude) + ) + time_from_prev_sec = distance_from_prev / planned_speed * 3600 # pracuji v km/h + marker_date_time = marker_date_time + timedelta(seconds=time_from_prev_sec) + + background_tasks.add_task( + download_weather_for_plan_marker, + marker_id=marker.id, date_time=marker_date_time + ) + previous_marker = marker diff --git a/src/utils/list_utils.py b/src/utils/list_utils.py index 4bd685f..a8d825c 100644 --- a/src/utils/list_utils.py +++ b/src/utils/list_utils.py @@ -12,3 +12,10 @@ def chunks(items: list | dict, n: int) -> typing.Generator: it = iter(items) for i in range(0, len(items), n): yield {k: items[k] for k in islice(it, n)} + + +def evenly_spaced_elements(lst: list, count: int) -> list: + if count > len(lst): + return lst + interval = (len(lst) - 1) / (count - 1) if count > 1 else 0 + return [lst[int(round(i * interval))] for i in range(count)]