poletani.cz/api
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Nova struktura DB pro data z OpenAIP API

Browse Source
This commit is contained in:
Michal Kváček
2024-04-29 06:44:33 +02:00
parent c2b066d371
commit c8138860ca
11 changed files with 342 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
alembic/versions/20240421-223250_reworked_airspace_and_airport_db__cbdb018c5490.py
+116
View File
@@ -0,0 +1,116 @@
"""reworked airspace and airport db structure
Revision ID: cbdb018c5490
Revises: 70152c47cb49
Create Date: 2024-04-21 22:32:50.058166
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
# revision identifiers, used by Alembic.
revision = 'cbdb018c5490'
down_revision = '70152c47cb49'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('airspace',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('country', sa.String(length=8), nullable=False),
sa.Column('name', sa.String(length=128), nullable=False),
sa.Column('type', sa.String(length=8), nullable=False),
sa.Column('lower_limit', sa.String(length=32), nullable=False),
sa.Column('upper_limit', sa.String(length=32), nullable=False),
sa.Column('center', mysql.JSON(), nullable=False),
sa.Column('radius', sa.Float(), nullable=True),
sa.Column('bounds', mysql.JSON(), nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
)
op.create_index(op.f('ix_airspace_country'), 'airspace', ['country'], unique=False)
op.create_index(op.f('ix_airspace_type'), 'airspace', ['type'], unique=False)
op.create_table('frequency',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('airport_id', sa.Integer(), nullable=True),
sa.Column('airspace_id', sa.Integer(), nullable=True),
sa.Column('frequency', sa.String(length=7), nullable=False),
sa.Column('name', sa.String(length=64), nullable=False),
sa.ForeignKeyConstraint(['airport_id'], ['airport.id'], ),
sa.ForeignKeyConstraint(['airspace_id'], ['airspace.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('runway',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('airport_id', sa.Integer(), nullable=False),
sa.Column('heading', sa.Integer(), nullable=False),
sa.Column('designator', sa.String(length=4), nullable=False),
sa.Column('surface', sa.Enum('asphalt', 'concrete', 'grass', 'other'), server_default='other', nullable=False),
sa.Column('length', sa.Integer(), nullable=True),
sa.Column('width', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['airport_id'], ['airport.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.drop_table('metar')
op.drop_table('license')
op.drop_index('ix_airspace_item_country', table_name='airspace_item')
op.drop_index('ix_airspace_item_type', table_name='airspace_item')
op.drop_index('name', table_name='airspace_item')
op.drop_table('airspace_item')
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('airspace_item',
sa.Column('id', mysql.INTEGER(display_width=11), autoincrement=True, nullable=False),
sa.Column('country', mysql.VARCHAR(length=8), nullable=False),
sa.Column('name', mysql.VARCHAR(length=128), nullable=False),
sa.Column('type', mysql.VARCHAR(length=8), nullable=False),
sa.Column('lower_limit', mysql.VARCHAR(length=32), nullable=False),
sa.Column('upper_limit', mysql.VARCHAR(length=32), nullable=False),
sa.Column('center', mysql.LONGTEXT(charset='utf8mb4', collation='utf8mb4_bin'), nullable=False),
sa.Column('radius', mysql.FLOAT(), nullable=True),
sa.Column('bounds', mysql.LONGTEXT(charset='utf8mb4', collation='utf8mb4_bin'), nullable=False),
sa.PrimaryKeyConstraint('id'),
mysql_collate='utf8mb4_general_ci',
mysql_default_charset='utf8mb4',
mysql_engine='InnoDB'
)
op.create_index('name', 'airspace_item', ['name'], unique=True)
op.create_index('ix_airspace_item_type', 'airspace_item', ['type'], unique=False)
op.create_index('ix_airspace_item_country', 'airspace_item', ['country'], unique=False)
op.create_table('license',
sa.Column('id', mysql.INTEGER(display_width=11), autoincrement=True, nullable=False),
sa.Column('name', mysql.VARCHAR(length=128), nullable=False),
sa.Column('number', mysql.VARCHAR(length=30), nullable=False),
sa.Column('valid_until', mysql.DATETIME(), nullable=False),
sa.Column('created_by_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=False),
sa.Column('created_at', mysql.DATETIME(), server_default=sa.text('current_timestamp()'), nullable=False),
sa.ForeignKeyConstraint(['created_by_id'], ['user.id'], name='license_ibfk_1'),
sa.PrimaryKeyConstraint('id'),
mysql_collate='utf8mb4_general_ci',
mysql_default_charset='utf8mb4',
mysql_engine='InnoDB'
)
op.create_table('metar',
sa.Column('id', mysql.INTEGER(display_width=11), autoincrement=True, nullable=False),
sa.Column('airport_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=False),
sa.Column('metar', mysql.TEXT(), nullable=False),
sa.Column('issued_at', mysql.DATETIME(), nullable=False),
sa.Column('deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), autoincrement=False, nullable=False),
sa.ForeignKeyConstraint(['airport_id'], ['airport.id'], name='metar_ibfk_1'),
sa.PrimaryKeyConstraint('id'),
mysql_collate='utf8mb4_general_ci',
mysql_default_charset='utf8mb4',
mysql_engine='InnoDB'
)
op.drop_table('runway')
op.drop_table('frequency')
op.drop_index(op.f('ix_airspace_type'), table_name='airspace')
op.drop_index(op.f('ix_airspace_country'), table_name='airspace')
op.drop_table('airspace')
# ### end Alembic commands ###
alembic/versions/20240422-234252_remove_old_airspace_columns_add__e07a5c0f1dcc.py
+32
View File
@@ -0,0 +1,32 @@
"""remove old airspace columns, add altitude unit
Revision ID: e07a5c0f1dcc
Revises: cbdb018c5490
Create Date: 2024-04-22 23:42:52.140389
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
# revision identifiers, used by Alembic.
revision = 'e07a5c0f1dcc'
down_revision = 'cbdb018c5490'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('airspace', sa.Column('altitude_unit', sa.String(length=8), nullable=False))
op.drop_column('airspace', 'center')
op.drop_column('airspace', 'radius')
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('airspace', sa.Column('radius', mysql.FLOAT(), nullable=True))
op.add_column('airspace', sa.Column('center', mysql.LONGTEXT(charset='utf8mb4', collation='utf8mb4_bin'), nullable=False))
op.drop_column('airspace', 'altitude_unit')
# ### end Alembic commands ###
alembic/versions/20240422-234917_add_alt_unit_for_lower_upper_limits_in__2012eca075c7.py
+32
View File
@@ -0,0 +1,32 @@
"""add alt unit for lower/upper limits in airspaces
Revision ID: 2012eca075c7
Revises: e07a5c0f1dcc
Create Date: 2024-04-22 23:49:17.725111
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
# revision identifiers, used by Alembic.
revision = '2012eca075c7'
down_revision = 'e07a5c0f1dcc'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('airspace', sa.Column('lower_alt_unit', sa.String(length=8), nullable=False))
op.add_column('airspace', sa.Column('upper_alt_unit', sa.String(length=8), nullable=False))
op.drop_column('airspace', 'altitude_unit')
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('airspace', sa.Column('altitude_unit', mysql.VARCHAR(length=8), nullable=False))
op.drop_column('airspace', 'upper_alt_unit')
op.drop_column('airspace', 'lower_alt_unit')
# ### end Alembic commands ###
alembic/versions/20240423-213918_add_table_for_reporting_points_03ba4dfea911.py
+45
View File
@@ -0,0 +1,45 @@
"""add table for reporting points
Revision ID: 03ba4dfea911
Revises: 2012eca075c7
Create Date: 2024-04-23 21:39:18.254033
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '03ba4dfea911'
down_revision = '2012eca075c7'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('reporting_point',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('openaip_id', sa.String(length=32), nullable=False),
sa.Column('name', sa.String(length=64), nullable=False),
sa.Column('gps_latitude', sa.Float(), nullable=True),
sa.Column('gps_longitude', sa.Float(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('airport_has_reporting_point',
sa.Column('airport_id', sa.Integer(), nullable=False),
sa.Column('reporting_point_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['airport_id'], ['airport.id'], ),
sa.ForeignKeyConstraint(['reporting_point_id'], ['reporting_point.id'], ),
sa.PrimaryKeyConstraint('airport_id', 'reporting_point_id')
)
op.add_column('airport', sa.Column('openaip_id', sa.String(length=32), nullable=False))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('airport', 'openaip_id')
op.drop_table('airport_has_reporting_point')
op.drop_table('reporting_point')
# ### end Alembic commands ###
alembic/versions/20240424-062448_remove_obsolete_airport_columns_f7ab4bdac987.py
Executable
+30
View File
@@ -0,0 +1,30 @@
"""remove obsolete airport columns
Revision ID: f7ab4bdac987
Revises: 03ba4dfea911
Create Date: 2024-04-24 06:24:48.219726
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
# revision identifiers, used by Alembic.
revision = 'f7ab4bdac987'
down_revision = '03ba4dfea911'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('airport', 'deleted')
op.drop_column('airport', 'is_public')
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('airport', sa.Column('is_public', mysql.TINYINT(display_width=1), server_default=sa.text('0'), autoincrement=False, nullable=False))
op.add_column('airport', sa.Column('deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), autoincrement=False, nullable=False))
# ### end Alembic commands ###
docker-compose.yml
+4 -9
View File
@@ -3,10 +3,8 @@ services:
image: mariadb:10.11
ports:
- 3306:3306
environment:
- MYSQL_USER=root
- MYSQL_ROOT_PASSWORD=root
- MYSQL_DATABASE=ull_tracker
env_file:
- .env
volumes:
# - ./db/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
- db-data:/var/lib/mysql:rw
@@ -37,11 +35,8 @@ services:
scrapy:
build:
context: ../crawlers
environment:
- MYSQL_USER=root
- MYSQL_PASSWORD=root
- MYSQL_DATABASE=ull_tracker
- MYSQL_HOST=db
env_file:
- .env
volumes:
- db-data:/var/lib/mysql:rw
- ../crawlers:/app
src/database/models.py
+45 -34
View File
@@ -67,17 +67,25 @@ copilot_has_photo = Table(
)
class AirspaceItem(BaseModel):
__tablename__ = "airspace_item"
airport_has_reporting_point = Table(
"airport_has_reporting_point",
BaseModel.metadata,
Column("airport_id", ForeignKey("airport.id"), primary_key=True),
Column("reporting_point_id", ForeignKey("reporting_point.id"), primary_key=True),
)
class Airspace(BaseModel):
__tablename__ = "airspace"
id: Mapped[int] = mapped_column(primary_key=True)
country: Mapped[str] = mapped_column(String(8), nullable=False, index=True)
name: Mapped[str] = mapped_column(String(128), nullable=False, unique=True)
type: Mapped[str] = mapped_column(String(8), nullable=False, index=True)
lower_limit: Mapped[str] = mapped_column(String(32), nullable=False)
lower_alt_unit: Mapped[str] = mapped_column(String(8), nullable=False)
upper_limit: Mapped[str] = mapped_column(String(32), nullable=False)
center: Mapped[List[dict]] = mapped_column(JSON())
radius: Mapped[float] = mapped_column(Float, nullable=True)
upper_alt_unit: Mapped[str] = mapped_column(String(8), nullable=False)
bounds: Mapped[list[tuple[float, float]]] = mapped_column(JSON())
@@ -111,21 +119,51 @@ class Airport(BaseModel):
id: Mapped[int] = mapped_column(primary_key=True)
name: Mapped[str] = mapped_column(String(128), nullable=False)
openaip_id: Mapped[str] = mapped_column(String(32), nullable=False, unique=True)
icao_code: Mapped[str] = mapped_column(String(8), nullable=False)
gps_latitude: Mapped[float] = mapped_column(Float, nullable=True)
gps_longitude: Mapped[float] = mapped_column(Float, nullable=True)
elevation: Mapped[int] = mapped_column(Integer, nullable=True)
airport_type: Mapped[str] = mapped_column(Enum("airport", "ull", "heliport"), nullable=False, server_default='airport') # noqa
use_in_gpx_guess: Mapped[bool] = mapped_column(Boolean, server_default='1')
is_public: Mapped[bool] = mapped_column(Boolean, server_default='0')
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
created_by_id: Mapped[int] = mapped_column(Integer, ForeignKey('user.id'), nullable=True) # automaticky import nebude mit ID # noqa
deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default='0')
metars: Mapped['Metar'] = relationship()
created_by: Mapped['User'] = relationship()
class Runway(BaseModel):
__tablename__ = "runway"
id: Mapped[int] = mapped_column(primary_key=True)
airport_id: Mapped[int] = mapped_column(Integer, ForeignKey('airport.id'))
heading: Mapped[int] = mapped_column(Integer, nullable=False)
designator: Mapped[str] = mapped_column(String(4), nullable=False)
surface: Mapped[str] = mapped_column(Enum("asphalt", "concrete", "grass", "other"), nullable=False, server_default="other") # noqa
length: Mapped[int] = mapped_column(Integer, nullable=True)
width: Mapped[int] = mapped_column(Integer, nullable=True)
class Frequency(BaseModel):
__tablename__ = "frequency"
id: Mapped[int] = mapped_column(primary_key=True)
airport_id: Mapped[int] = mapped_column(Integer, ForeignKey('airport.id'), nullable=True)
airspace_id: Mapped[int] = mapped_column(Integer, ForeignKey('airspace.id'), nullable=True)
frequency: Mapped[str] = mapped_column(String(7), nullable=False)
name: Mapped[str] = mapped_column(String(64), nullable=False)
class ReportingPoint(BaseModel):
__tablename__ = "reporting_point"
id: Mapped[int] = mapped_column(primary_key=True)
openaip_id: Mapped[str] = mapped_column(String(32), nullable=False)
name: Mapped[str] = mapped_column(String(64), nullable=False)
gps_latitude: Mapped[float] = mapped_column(Float, nullable=True)
gps_longitude: Mapped[float] = mapped_column(Float, nullable=True)
class PointOfInterestType(BaseModel):
__tablename__ = "point_of_interest_type"
@@ -370,32 +408,6 @@ class Copilot(BaseModel):
title_photo: Mapped['Photo'] = relationship(foreign_keys=[title_photo_id])
class Metar(BaseModel):
__tablename__ = "metar"
id: Mapped[int] = mapped_column(primary_key=True)
airport_id: Mapped[int] = mapped_column(Integer, ForeignKey('airport.id'))
metar: Mapped[str] = mapped_column(Text, nullable=False)
issued_at: Mapped[datetime] = mapped_column(DateTime)
deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default='0')
airport: Mapped['Airport'] = relationship()
class License(BaseModel):
__tablename__ = "license"
id: Mapped[int] = mapped_column(primary_key=True)
name: Mapped[str] = mapped_column(String(128), nullable=False)
number: Mapped[str] = mapped_column(String(30), nullable=False)
valid_until: Mapped[datetime] = mapped_column(DateTime, nullable=False)
created_by_id: Mapped[int] = mapped_column(Integer, ForeignKey('user.id'))
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
user: Mapped['User'] = relationship()
created_by: Mapped['User'] = relationship()
class User(BaseModel):
__tablename__ = "user"
excluded_columns_in_dict = ('password_hashed',)
@@ -410,7 +422,6 @@ class User(BaseModel):
password_hashed: Mapped[str] = mapped_column(String(60), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
licences: Mapped[Set['License']] = relationship()
flights: Mapped[Set['Flight']] = relationship()
organizations: Mapped[Set['Organization']] = relationship(secondary=user_is_in_organization)
src/external/openair_parser.py
Vendored
+12 -12
View File
@@ -14,7 +14,7 @@ from external.gps_utils import get_distance, get_bearing, EARTH_RADIUS
# https://pyopenair.readthedocs.io/en/latest/openair.html
# http://www.winpilot.com/usersguide/userairspace.asp
class AirspaceItemType(enum.Enum):
class AirspaceType(enum.Enum):
AIRSPACE_A = "A"
AIRSPACE_B = "B"
AIRSPACE_C = "C"
@@ -47,8 +47,8 @@ class Coordinates:
@dataclasses.dataclass
class AirspaceItem:
type: AirspaceItemType
class Airspace:
type: AirspaceType
info: str = ""
upper_limit: str = None
lower_limit: str = None
@@ -107,7 +107,7 @@ class OpenAirParser:
def __init__(self, file: Path | str):
self.file = None
self.context_type: Optional[AirspaceItemType] = None
self.context_type: Optional[AirspaceType] = None
self.context = {}
self.file = file
@@ -116,7 +116,7 @@ class OpenAirParser:
if not self.context_type:
return
data = AirspaceItem(type=self.context_type, **self.context)
data = Airspace(type=self.context_type, **self.context)
self.context_type = None
self.context = {}
@@ -194,19 +194,19 @@ class OpenAirParser:
pass
case 'AC':
try:
self.context_type = AirspaceItemType(value)
self.context_type = AirspaceType(value)
except ValueError as e:
print(f"Neznam {e}, ignoruji")
return
case 'AN':
if self.context_type == AirspaceItemType.AIRSPACE_G:
if self.context_type == AirspaceType.AIRSPACE_G:
match value[0:2]:
case 'PG':
self.context_type = AirspaceItemType.PARAGLIDING_ZONE
self.context_type = AirspaceType.PARAGLIDING_ZONE
case 'LK':
self.context_type = AirspaceItemType.ATZ
self.context_type = AirspaceType.ATZ
case _:
self.context_type = AirspaceItemType.TRAGA
self.context_type = AirspaceType.TRAGA
self.context['info'] = value
case 'AL':
@@ -249,7 +249,7 @@ async def download_airspace(source, destination):
async def save_airspace(destination):
parser = OpenAirParser(destination)
async with get_session() as db:
await db.execute(delete(models.AirspaceItem))
await db.execute(delete(models.Airspace))
for item in parser.parse():
# if 'CTR KARLOVY VARY' not in item.info:
@@ -261,7 +261,7 @@ async def save_airspace(destination):
# print("---" * 10)
async with get_session() as db:
await models.AirspaceItem.create(db, {
await models.Airspace.create(db, {
"country": "cz",
"name": item.info,
"type": item.type.value,
src/graphql_schema/dataloaders/multi_models.py
+8
View File
@@ -23,6 +23,14 @@ flight_copilots_dataloader = DataLoader(
extra_join=[models.Copilot.flights]).load,
cache=False)
reporting_points_dataloader = DataLoader(
load_fn=MultiModelsDataloader(
models.ReportingPoint,
relationship_column=models.airport_has_reporting_point.c.airport_id,
extra_join=[models.airport_has_reporting_point]
).load, cache=False
)
flights_by_copilot_dataloader = DataLoader(
load_fn=MultiModelsDataloader(
models.Flight,
src/graphql_schema/entities/airspace.py
+3 -3
View File
@@ -16,11 +16,11 @@ class AirspaceQueries:
filters = []
if country:
filters.append(models.AirspaceItem.country == country)
filters.append(models.Airspace.country == country)
if types:
filters.append(models.AirspaceItem.type.in_(types))
filters.append(models.Airspace.type.in_(types))
return await BaseQueryResolver(Airspace, models.AirspaceItem).get_list(
return await BaseQueryResolver(Airspace, models.Airspace).get_list(
info.context.user_id,
country=country,
filters=filters
src/graphql_schema/entities/types/types.py
+15 -8
View File
@@ -14,7 +14,7 @@ from graphql_schema.dataloaders.multi_models import (
photos_dataloader, flights_by_aircraft_dataloader, users_in_organization_dataloader,
aircrafts_from_organization_dataloader, user_organizations_dataloader, flights_by_event_dataloader,
flights_by_copilot_dataloader, public_flights_by_event_dataloader, public_flights_by_copilot_dataloader,
photo_copilots_dataloader, photos_aircraft_dataloader, copilots_in_photo_dataloader, flight_plan_markers_dataloader
photo_copilots_dataloader, photos_aircraft_dataloader, copilots_in_photo_dataloader, flight_plan_markers_dataloader, reporting_points_dataloader
)
from graphql_schema.dataloaders.single_model import (
poi_dataloader, poi_type_dataloader, event_dataloader, aircraft_dataloader, airport_dataloader,
@@ -46,9 +46,19 @@ class GPXTrack:
avg_altitude: float
@strawberry_sqlalchemy_type(models.ReportingPoint)
class ReportingPoint:
pass
@strawberry_sqlalchemy_type(models.Airport)
class Airport:
pass
reporting_points: List[ReportingPoint] = strawberry.field(
resolver=lambda root: reporting_points_dataloader.load(root.id)
)
# TODO: loader na ikonu
# TODO: loader na RWY a frekvence
@strawberry.type
@@ -58,15 +68,12 @@ class Airspace:
country: str
type: str
lower_limit: str
lower_alt_unit: str
upper_alt_unit: str
upper_limit: str
center: Optional[tuple[float, float]]
radius: Optional[float]
bounds: list[tuple[float, float]]
map_center: Optional[Point] = strawberry.field(
resolver=lambda root: Point(lat=root.center[0], lng=root.center[1]) if root.center else None
)
map_bounds: list[Point] = strawberry.field(
resolver=lambda root: [Point(lat=point[0], lng=point[1]) for point in root.bounds]
resolver=lambda root: [Point(lat=point[1], lng=point[0]) for point in root.bounds]
)