{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "71fe19c0-ebd2-42bb-a712-5b9da548e98d", "metadata": {}, "outputs": [], "source": [ "CONTEST_ID = \"IARU-FIELD-DAY\"\n", "ADI_LOG_FILENAME = \"DL0ABT-fd26.adi\"\n", "# https://www.openstreetmap.org/?mlat=52.38638&mlon=13.89543#map=15/52.38638/13.89543\n", "CONTEST_QTH = (52.38638, 13.89543) # Wiese im Wald" ] }, { "cell_type": "code", "execution_count": null, "id": "a2cdf962-9adf-4712-9673-ae5b329cc165", "metadata": {}, "outputs": [], "source": [ "import adif_io\n", "import os\n", "import os.path\n", "import requests\n", "import json\n", "import math\n", "\n", "if not os.path.isdir(\"cache\"):\n", " os.mkdir(\"cache\")\n", "\n", "def qsos_with_lat_and_long(filename):\n", " qsos, _headers = adif_io.read_from_file(filename)\n", "\n", " def is_portable_station(call:str) -> bool:\n", " for suffix in [\"/p\", \"/P\", \"/m\", \"/M\", \"/mm\", \"/MM\", \"/am\", \"/AM\"]:\n", " if call.endswith(suffix):\n", " return True\n", " return False\n", "\n", " dupecheck = set()\n", "\n", " # Grab info from hamqth.com:\n", " for qso in sorted(qsos, key=lambda qso: adif_io.time_on(qso)):\n", " try:\n", " call_sign = qso[\"CALL\"]\n", " json_file_name = f\"cache/{call_sign.replace('/', '-')}_hamqth.json\"\n", " if not os.path.isfile(json_file_name):\n", " res = requests.get(f\"https://www.hamqth.com/dxcc_json.php?callsign={call_sign}\")\n", " res.raise_for_status()\n", " with open(json_file_name, \"wb\") as jf:\n", " jf.write(res.content)\n", " try:\n", " with open(json_file_name, \"rb\") as jf:\n", " json_info = json.load(jf)\n", " except:\n", " print(f\"Problem with file {json_file_name}\")\n", " raise\n", " if \"LAT\" in qso and \"LON\" in qso and \"DXCC\" in qso and \"DXCC_NAME\" in qso and \"CONT\" in qso:\n", " pass\n", " else:\n", " qso[\"LAT\"] = adif_io.location_from_degrees(float(json_info[\"lat\"]), True)\n", " qso[\"LON\"] = adif_io.location_from_degrees(float(json_info[\"lng\"]), False)\n", " if \"DXCC\" not in qso:\n", " qso[\"DXCC\"] = json_info[\"adif\"]\n", " if \"dxcc_name\" not in qso:\n", " qso[\"dxcc_name\"] = json_info[\"name\"]\n", " dupe_key = f\"{qso['CALL']} {qso['BAND']}\"\n", " if \"CONT\" not in qso:\n", " qso[\"CONT\"] = json_info[\"continent\"]\n", " \n", " if dupe_key in dupecheck:\n", " print(f\"{dupe_key:14} QSO at {adif_io.time_on(qso)} \"\n", " f\"{'RUN' if qso['APP_N1MM_ISRUNQSO'] == '1' else \"S/P\"} {qso[\"OPERATOR\"]:6s} is dupe.\")\n", " qso[\"qso_points\"] = 0\n", " else:\n", " dupecheck.add(dupe_key)\n", " if is_portable_station(call_sign):\n", " if qso[\"CONT\"] == \"EU\":\n", " qso[\"qso_points\"] = 4\n", " else:\n", " qso[\"qso_points\"] = 6\n", " else:\n", " if qso[\"CONT\"] == \"EU\":\n", " qso[\"qso_points\"] = 2\n", " else:\n", " qso[\"qso_points\"] = 3\n", " \n", " if \"APP_N1MM_POINTS\" in qso and int(qso[\"qso_points\"]) != int(qso[\"APP_N1MM_POINTS\"]):\n", " raise RuntimeError(f\"Conflict qso_points vs APP_N1MM_POINTS: QSO with {qso['CALL']} on {qso['BAND']}: {qso}\")\n", " except:\n", " print(f\"Problem with qso {qso}\")\n", " raise\n", "\n", " missing_qsos = [qso for qso in qsos if \"LAT\" not in qso or \"LON\" not in qso]\n", " if 0 < len(missing_qsos):\n", " raise RuntimeError(\"QSOs without location data: \", missing_qsos)\n", "\n", " return [qso for qso in qsos if \"LAT\" in qso and \"LON\" in qso]\n", "\n", "qsos_unsorted = qsos_with_lat_and_long(ADI_LOG_FILENAME)" ] }, { "cell_type": "code", "execution_count": null, "id": "b867cd05-a777-4c51-9008-2cdb921b2894", "metadata": {}, "outputs": [], "source": [ "year = adif_io.time_on(qsos_unsorted[0]).year" ] }, { "cell_type": "code", "execution_count": null, "id": "b6540273-13e3-41f8-9311-9f7a92a5e43a", "metadata": {}, "outputs": [], "source": [ "total_qso_points = sum((int(qso[\"qso_points\"]) for qso in qsos_unsorted))\n", "total_qso_points" ] }, { "cell_type": "code", "execution_count": null, "id": "6d2cfb7d-c4b7-4d75-ad0b-e8305a171903", "metadata": {}, "outputs": [], "source": [ "def find_station_callsign():\n", " station_callsigns = set((qso[\"STATION_CALLSIGN\"] for qso in qsos_unsorted))\n", " if 1 == len(station_callsigns):\n", " return station_callsigns.pop()\n", " else:\n", " raise ValueError(f\"Not exactly one callsign found: {station_callsigns}\")\n", "station_callsign = find_station_callsign()\n", "station_callsign" ] }, { "cell_type": "code", "execution_count": null, "id": "a552019e-aa93-40ec-9ef9-84615bff6791", "metadata": {}, "outputs": [], "source": [ "from datetime import date, datetime, timedelta, UTC\n", "\n", "def calculate_contest_time(year):\n", " first_day_in_june = date(year, 6, 1)\n", " first_saturday_in_june = first_day_in_june\n", " SATURDAY = 5\n", " while(first_saturday_in_june.weekday() != SATURDAY):\n", " first_saturday_in_june += timedelta(days=1)\n", " contest_start_day = first_saturday_in_june\n", " contest_start = datetime(year=contest_start_day.year,\n", " month=contest_start_day.month,\n", " day=contest_start_day.day,\n", " hour=15,\n", " tzinfo=UTC)\n", " contest_end = contest_start + timedelta(days=1)\n", " return contest_start, contest_end\n", "\n", "start_of_contest, end_of_contest = calculate_contest_time(year)\n", "print(\"Contest took place \", str(start_of_contest), \"until\", str(end_of_contest))" ] }, { "cell_type": "code", "execution_count": null, "id": "2658d68e-5d87-4eab-98b1-7435363de881", "metadata": {}, "outputs": [], "source": [ "# DXCCs reached:\n", "from collections import Counter\n", "\n", "def print_dxccs_reached():\n", " dxcc_reached_count = Counter()\n", " for qso in qsos_unsorted:\n", " try:\n", " dxcc_reached_count[qso[\"dxcc_name\"]] += 1\n", " except:\n", " print(f\"Problem with QSO: {qso}\")\n", " raise\n", " dxccs_reached = sorted(set((qso[\"dxcc_name\"] for qso in qsos_unsorted)))\n", " for dxcc in sorted(dxcc_reached_count.keys()):\n", " print(f\"{dxcc}: {dxcc_reached_count[dxcc]}\")\n", " print(f\"\\nTotal DXCCs: {len(dxcc_reached_count)}\")\n", "print_dxccs_reached()" ] }, { "cell_type": "code", "execution_count": null, "id": "f240b4a7-902d-44a4-bef8-b6dcefb71c5c", "metadata": {}, "outputs": [], "source": [ "# How often did we reach which band/call combination:\n", "\n", "def multi(qso) -> str:\n", " band = int(qso['Band'][:-1])\n", " return band, qso['dxcc_name']\n", "\n", "multi_count = Counter()\n", "for qso in qsos_unsorted:\n", " multi_count[multi(qso)] += 1\n", "multis_raw = set((multi(qso) for qso in qsos_unsorted))\n", "\n", "total_multis = len(multis_raw)\n", "\n", "# multis = sorted(((mul, f\" {mul[1]}: {multi_count[mul]}\") for mul in multis_raw))\n", "last_band = None\n", "for mul in sorted(multis_raw):\n", " if last_band != mul[0]:\n", " if last_band is not None:\n", " print(\"\")\n", " print(f\"{mul[0]:3d} m Band:\")\n", " last_band = mul[0]\n", " print(f\" {mul[1]}: {multi_count[mul]}\")\n", "\n", "print(f\"\\nTotal DXCC-Band Kombinations: {total_multis}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "debdcaa7-2255-4988-ae5f-1722325d9a12", "metadata": {}, "outputs": [], "source": [ "qsos = sorted(qsos_unsorted, key=adif_io.time_on)" ] }, { "cell_type": "code", "execution_count": null, "id": "d818aaf3-2ea0-470e-b2ad-ba9aacb4a5b4", "metadata": {}, "outputs": [], "source": [ "from functools import reduce\n", "\n", "# QSO points\n", "band_call2points = dict(\n", " ((f\"{qso['CALL']} {qso['BAND']}\", int(qso[\"qso_points\"])) for qso in qsos_unsorted if 0 < int(qso[\"qso_points\"]))\n", ")\n", "qso_points = reduce(lambda sum, points: sum + points, band_call2points.values(), 0)\n", "# print([qso for qso in qsos_unsorted if qso[\"qso_points\"] == 6])\n", "num_of_dxcc_bands = len(multis_raw)\n", "claimed_score = num_of_dxcc_bands * qso_points\n", "f\"band+dxcc combinations: {num_of_dxcc_bands}, qso_points: {qso_points}, claimed score {claimed_score}\"" ] }, { "cell_type": "code", "execution_count": null, "id": "cd73843c-a813-4ad0-9a5a-322375724419", "metadata": {}, "outputs": [], "source": [ "print(f\"{len(set((qso['CALL'] for qso in qsos_unsorted)))} individual calls reached.\")\n", "print(f\"{len(set((qso['CALL'] for qso in qsos_unsorted \\\n", " if qso['CALL'].endswith('/P'))))} individual /P calls reached.\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "ab042289-9b5b-4f2c-bbdd-5e034f03960a", "metadata": {}, "outputs": [], "source": [ "import math\n", "from collections import defaultdict\n", "\n", "NO_QSO_WITHOUT_LEAVING_SEAT_SECONDS = 600\n", "ASSUMED_ACTIVITY_BEFORE_FIRST_QSO_AFTER_RETURNING_TO_SEAT = 180\n", "\n", "def calculate_time_in_seat() -> float:\n", " time_in_seat_seconds = 0\n", " last_active_time = None\n", " last_activity_start = None\n", " last_operator_seen = None\n", " last_band_seen = None\n", " qsos_this_period = 0\n", " op2time_in_seat_n_qsos = defaultdict(lambda: (timedelta(seconds=0), 0))\n", " def book():\n", " time_in_seat_n_qsos = op2time_in_seat_n_qsos[last_operator_seen]\n", " op2time_in_seat_n_qsos[last_operator_seen] = (\n", " time_in_seat_n_qsos[0] + (last_active_time-last_activity_start),\n", " time_in_seat_n_qsos[1] + qsos_this_period\n", " )\n", " print(\n", " f\"{last_operator_seen:6s}: \"\n", " f\"{last_activity_start.astimezone().strftime('%H:%M')} - {last_active_time.astimezone().strftime('%H:%M %Z')}: \"\n", " f\"{str(last_active_time-last_activity_start)[:-3]}h {qsos_this_period:3d} QSOs \"\n", " f\"{3600 * qsos_this_period / (last_active_time-last_activity_start).total_seconds():4.1f} QSOs/h \"\n", " f\"{last_band_seen}\"\n", " )\n", "\n", " first_qso = True\n", " for qso in qsos:\n", " active_time = adif_io.time_on(qso)\n", " if last_active_time is not None:\n", " delta = (active_time - last_active_time).total_seconds()\n", " if delta < NO_QSO_WITHOUT_LEAVING_SEAT_SECONDS:\n", " time_in_seat_seconds += delta\n", " qso[\"time_in_seat\"] = delta\n", " if qso[\"OPERATOR\"] != last_operator_seen or qso[\"BAND\"] != last_band_seen:\n", " book()\n", " qsos_this_period = 0\n", " last_activity_start = last_active_time\n", " else:\n", " book()\n", " new_last_activity_start = active_time - \\\n", " timedelta(seconds=ASSUMED_ACTIVITY_BEFORE_FIRST_QSO_AFTER_RETURNING_TO_SEAT)\n", " print(\n", " f\"Idle time \"\n", " f\"{str(new_last_activity_start - last_active_time)[:-3]}h: \"\n", " f\"{last_active_time.astimezone().strftime('%H:%M %Z')} - \"\n", " f\"{new_last_activity_start.astimezone().strftime('%H:%M %Z')}\"\n", " )\n", " last_activity_start = new_last_activity_start\n", " qsos_this_period = 0\n", " time_in_seat_seconds += ASSUMED_ACTIVITY_BEFORE_FIRST_QSO_AFTER_RETURNING_TO_SEAT\n", " qso[\"time_in_seat\"] = ASSUMED_ACTIVITY_BEFORE_FIRST_QSO_AFTER_RETURNING_TO_SEAT\n", " else:\n", " if first_qso:\n", " assumed_start_of_operation = \\\n", " active_time - timedelta(seconds=ASSUMED_ACTIVITY_BEFORE_FIRST_QSO_AFTER_RETURNING_TO_SEAT)\n", " print(\n", " f\"Idle time \"\n", " f\"{str( assumed_start_of_operation - start_of_contest)[:-3]}h: \"\n", " f\"{start_of_contest.astimezone().strftime('%H:%M %Z')} - \"\n", " f\"{assumed_start_of_operation.astimezone().strftime('%H:%M %Z')}\"\n", " )\n", " last_activity_start = active_time - timedelta(seconds=ASSUMED_ACTIVITY_BEFORE_FIRST_QSO_AFTER_RETURNING_TO_SEAT)\n", " qsos_this_period = 0\n", " time_in_seat_seconds += ASSUMED_ACTIVITY_BEFORE_FIRST_QSO_AFTER_RETURNING_TO_SEAT\n", " qso[\"time_in_seat\"] = ASSUMED_ACTIVITY_BEFORE_FIRST_QSO_AFTER_RETURNING_TO_SEAT\n", " last_operator_seen = qso[\"OPERATOR\"]\n", " last_band_seen = qso[\"BAND\"]\n", " last_active_time = active_time\n", " qsos_this_period += 1\n", " first_qso = False\n", " book()\n", " return time_in_seat_seconds, op2time_in_seat_n_qsos\n", "\n", "time_in_seat_seconds, op2time_in_seat_n_qsos = calculate_time_in_seat()\n", "\n", "print(\"\\nSummary:\")\n", "for op in sorted(op2time_in_seat_n_qsos.keys()):\n", " time_in_seat_n_qsos = op2time_in_seat_n_qsos[op]\n", " print(\n", " f\"{op:6s}: \"\n", " f\"{str(time_in_seat_n_qsos[0])[0:-3]}h \"\n", " f\"{time_in_seat_n_qsos[1]:3d} QSOs\"\n", " f\"{3600 * time_in_seat_n_qsos[1] / time_in_seat_n_qsos[0].total_seconds():5.1f} QSOs/h\"\n", " )\n", "\n", "full_hours = math.floor(time_in_seat_seconds / 3600)\n", "minutes = time_in_seat_seconds / 60 - full_hours * 60\n", "time_in_seat = f\"{full_hours}:{minutes:02.0f}\"\n", "qso_rate_per_hour = f\"{len(qsos) / (time_in_seat_seconds / 3600):3.1f}\"\n", "print(f\"\\nTotal time in seat: {time_in_seat}, average QSO rate {qso_rate_per_hour}\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "0bdd4abb-bbb7-4666-9449-30e7d54e0a3e", "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "import geopandas\n", "import pandas as pd\n", "from math import sqrt\n", "from cartopy import crs as ccrs\n", "\n", "# Beam projection for Berlin\n", "berlin_beam_crs = ccrs.AzimuthalEquidistant(central_latitude=CONTEST_QTH[0], central_longitude=CONTEST_QTH[1]).proj4_init\n", "\n", "qsos_with_qths = [qso for qso in qsos \\\n", " if adif_io.degrees_from_location(qso[\"LON\"]) != 0 and \\\n", " adif_io.degrees_from_location(qso[\"LAT\"]) != 0]\n", "\n", "all_qths_df = pd.DataFrame({\n", " \"call\": [qso[\"CALL\"] for qso in qsos_with_qths],\n", " \"lon\": [adif_io.degrees_from_location(qso[\"LON\"]) for qso in qsos_with_qths],\n", " \"lat\": [adif_io.degrees_from_location(qso[\"LAT\"]) for qso in qsos_with_qths]\n", "})\n", "\n", "all_qths = geopandas.GeoDataFrame(\n", " all_qths_df,\n", " geometry=geopandas.points_from_xy(all_qths_df.lon, all_qths_df.lat),\n", " crs=\"EPSG:4326\"\n", ").to_crs(berlin_beam_crs)\n", "\n", "# You need to obtain the public domain file 110m_cultural.zip, typically from \n", "# https://www.naturalearthdata.com/downloads/110m-cultural-vectors/\n", "# resp https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/cultural/110m_cultural.zip\n", "earth = geopandas.read_file(\"110m_cultural.zip\", layer='ne_110m_admin_0_countries_lakes').to_crs(berlin_beam_crs)\n", "\n", "# Possibility to zoom in on the places really worked. Set to None to show all:\n", "xmin = xmax = ymin = ymax = None\n", "\n", "xmin = 0\n", "xmax = 0\n", "ymin = 0\n", "ymax = 0\n", "odx = 0\n", "for p in all_qths[[\"geometry\"]].to_numpy():\n", " x = p[0].x\n", " y = p[0].y\n", " if x < xmin:\n", " xmin = x\n", " elif xmax < x:\n", " xmax = x\n", " if y < ymin:\n", " ymin = y\n", " elif ymax < y:\n", " ymax = y\n", " dx = sqrt(x*x + y*y)\n", " if odx < dx:\n", " odx = dx\n", "\n", "xmin -= 200e3\n", "xmax += 200e3\n", "ymin -= 200e3\n", "ymax += 200e3\n", "\n", "ax = earth.plot(\n", " figsize=(16,16 * (ymax-ymin)/(xmax-xmin) if xmin is not None else 16),\n", " color='lightgrey', edgecolor='black'\n", ")\n", "all_qths.plot(ax=ax, markersize=40, color=\"darkgreen\")\n", "ax.set_title(f\"{station_callsign} CWFD {year}, {len(qsos)} QSOs, {time_in_seat} hours time in seat, {qso_rate_per_hour} QSOs / hour, ODX = {odx*1e-3:.0f} km\")\n", "# To zoom in on wherever the data was:\n", "if xmin:\n", " ax.set_xlim(left=xmin, right=xmax)\n", " ax.set_ylim(bottom=ymin, top=ymax)\n", "pass" ] }, { "cell_type": "code", "execution_count": null, "id": "96084122-7590-4347-a40a-c5101bf706b9", "metadata": { "scrolled": true }, "outputs": [], "source": [ "qso_df = pd.DataFrame(qsos)\n", "qso_df.info()" ] }, { "cell_type": "code", "execution_count": null, "id": "2ef9780f-fa4f-4d21-848f-1b3b5dea4e99", "metadata": {}, "outputs": [], "source": [ "g_by_band = qso_df.groupby(qso_df[\"BAND\"])\n", "\n", "# I want these to be sorted by wavelength, so convert to int, sort, and convert back to string with trailing \"M\".\n", "bands = [f\"{b}M\" for b in sorted(set((int(qso[\"BAND\"][0:-1]) for qso in qsos)))]\n", "\n", "band_df = pd.DataFrame(\n", " {\"QSO Count\": [len(g_by_band.get_group(band)) for band in bands]},\n", " index=bands\n", ")\n", "band_df.plot(y=\"QSO Count\", kind=\"pie\", figsize=(7,7), title=f\"DL0ABT/p CWFD {year}, QSOs by bands\", legend=False)\n", "band_df" ] }, { "cell_type": "code", "execution_count": null, "id": "866c8cc2-0a64-489e-86c8-b02ec5766c73", "metadata": {}, "outputs": [], "source": [ "def qso_points_for_band(band: str) -> int:\n", " return reduce(lambda x, y: int(x) + int(y), g_by_band.get_group(band)[\"QSO_POINTS\"])\n", "\n", "points_band_df = pd.DataFrame(\n", " {\"QSO Points\": [qso_points_for_band(band) for band in bands]},\n", " index=bands\n", ")\n", "\n", "points_band_df.plot(y=\"QSO Points\", kind=\"pie\", figsize=(7,7), title=f\"DL0ABT/p CWFD {year}, QSO points by band\", legend=False)\n", "points_band_df" ] }, { "cell_type": "code", "execution_count": null, "id": "8b794f90-0a05-476e-b474-a3830bb392bb", "metadata": {}, "outputs": [], "source": [ "points_per_qso_df = pd.DataFrame(\n", " {\"Avg. points per QSO\": [\n", " points_band_df.loc[band][\"QSO Points\"] / band_df.loc[band][\"QSO Count\"]\n", " for band in bands\n", " ]},\n", " index=bands\n", ")\n", "points_per_qso_df.plot(y=\"Avg. points per QSO\", kind=\"bar\", figsize=(7,7), title=f\"DL0ABT/p CWFD {year}, avg. QSO points per QSO\", legend=False)\n", "points_per_qso_df" ] }, { "cell_type": "code", "execution_count": null, "id": "99998286-da1e-4a58-bcbc-545e29d66479", "metadata": {}, "outputs": [], "source": [ "g_by_cont = qso_df.groupby(qso_df[\"CONT\"])\n", "\n", "cgs = [(cont, len(g)) for cont, g in g_by_cont]\n", "\n", "cont_df = pd.DataFrame(\n", " {\"QSO Count\": [cg[1] for cg in cgs]},\n", " index=[cg[0] for cg in cgs]\n", ")\n", "cont_df.plot(y=\"QSO Count\", kind=\"pie\", figsize=(7,7), title=f\"DL0ABT/p CWFD {year}, QSOs by continent\", legend=False)\n", "cont_df" ] }, { "cell_type": "code", "execution_count": null, "id": "e69cceda-15e6-4da1-a227-700ae51de654", "metadata": {}, "outputs": [], "source": [ "\n", "band_dxcc_df = pd.DataFrame(\n", " {\"DXCC Count\": [len(g_by_band.get_group(band).groupby(\"DXCC\")) for band in bands]},\n", " index=bands\n", ")\n", "\n", "band_dxcc_df.plot(y=\"DXCC Count\", kind=\"bar\", figsize=(7,7), title=f\"DL0ABT/p CWFD {year}, DXCCs reached by band\", legend=False)\n", "\n", "band_dxcc_df" ] }, { "cell_type": "code", "execution_count": null, "id": "6be4b284-2782-43f0-9606-7ddddb26693a", "metadata": {}, "outputs": [], "source": [ "from datetime import timedelta\n", "from functools import reduce\n", "\n", "def get_time_in_seat(band: str):\n", " accumulated_time_in_seat = 0.0\n", " for i, qso in g_by_band.get_group(band).iterrows():\n", " accumulated_time_in_seat += float(qso[\"TIME_IN_SEAT\"])\n", " return accumulated_time_in_seat\n", "\n", "time_in_seat_df = pd.DataFrame(\n", " {\n", " \"time in seat\": [get_time_in_seat(band) for band in bands],\n", " \"time in seat/hours\": [get_time_in_seat(band)/3600 for band in bands],\n", " },\n", " index=bands\n", ")\n", "\n", "time_in_seat_df.plot(y=\"time in seat\", kind=\"pie\", figsize=(7,7), title=f\"DL0ABT/p CWFD {year}, time in seat by band\", legend=False)\n", "time_in_seat_df, reduce(lambda tis, band: tis + (time_in_seat_df.loc[band][\"time in seat\"]), bands, 0.0)/3600, time_in_seat_seconds / 3600" ] }, { "cell_type": "code", "execution_count": null, "id": "1fa1b34d-ad5b-46bc-92fa-928f12cc20df", "metadata": {}, "outputs": [], "source": [ "qso_rate_df = pd.DataFrame(\n", " {\"QSOs per hour\": [\n", " band_df.loc[band][\"QSO Count\"] / (time_in_seat_df.loc[band][\"time in seat\"]/3600) for band in bands\n", " ]\n", " },\n", " index=bands\n", ")\n", "qso_rate_df.plot(\n", " y=\"QSOs per hour\",\n", " kind=\"bar\",\n", " figsize=(7,7),\n", " title=f\"DL0ABT/p CWFD {year}, QSOs per hour rate by band\",\n", " legend=False\n", ")\n", "qso_rate_df" ] }, { "cell_type": "code", "execution_count": null, "id": "0e80f2a4-8aaf-4c8c-bba1-5a8e5ee03617", "metadata": {}, "outputs": [], "source": [ "qso_point_rate_df = pd.DataFrame(\n", " {\"QSO points per hour\": [qso_points_for_band(band) / (time_in_seat_df.loc[band][\"time in seat\"]/3600) for band in bands]},\n", " index=bands\n", ")\n", "qso_point_rate_df.plot(\n", " y=\"QSO points per hour\",\n", " kind=\"bar\",\n", " figsize=(7,7),\n", " title=f\"DL0ABT/p CWFD {year}, QSO points per hour by band\",\n", " legend=False\n", ")\n", "qso_point_rate_df" ] }, { "cell_type": "code", "execution_count": null, "id": "1f0e7f5d-d086-44e6-baa7-640e6970d659", "metadata": {}, "outputs": [], "source": [ "dxcc_rate_df = pd.DataFrame(\n", " {\"DXCC per hour\": [band_dxcc_df.loc[band][\"DXCC Count\"] / (time_in_seat_df.loc[band][\"time in seat\"]/3600) for band in bands]},\n", " index=bands\n", ")\n", "dxcc_rate_df.plot(\n", " y=\"DXCC per hour\",\n", " kind=\"bar\",\n", " figsize=(7,7),\n", " title=f\"DL0ABT/p CWFD {year}, DXCCs rate per hour by band\",\n", " legend=False\n", ")\n", "dxcc_rate_df" ] }, { "cell_type": "code", "execution_count": null, "id": "47846614-8cc3-4d51-8338-2dac6bd5298d", "metadata": {}, "outputs": [], "source": [ "def points_missing_if_band_missing(band:str) -> int:\n", " dxcc_count_band = band_dxcc_df.loc[band][\"DXCC Count\"]\n", " qso_points_band = points_band_df.loc[band][\"QSO Points\"]\n", " points_without_band = (num_of_dxcc_bands - dxcc_count_band) * (qso_points - qso_points_band)\n", " return claimed_score - points_without_band\n", "\n", "points_missing_if_band_missing_df = pd.DataFrame(\n", " {\"Points missing if band missing\": [points_missing_if_band_missing(band) for band in bands]},\n", " index=bands\n", ")\n", "points_missing_if_band_missing_df.plot(\n", " y=\"Points missing if band missing\",\n", " kind=\"bar\",\n", " figsize=(7,7),\n", " title=f\"DL0ABT/p CWFD {year}, resultpoints missing if band missing\",\n", " legend=False\n", ")\n", "points_missing_if_band_missing_df" ] }, { "cell_type": "code", "execution_count": null, "id": "7c970645-1ad0-4d27-b411-358dca7bbfb3", "metadata": {}, "outputs": [], "source": [ "qso_df[\"accumulated_time_in_seat_by_band\"] = None\n", "\n", "def set_time_in_seat_by_band(band: str):\n", " accumulated_time_in_seat = 0.0\n", " for i, qso in g_by_band.get_group(band).iterrows():\n", " accumulated_time_in_seat += float(qso[\"TIME_IN_SEAT\"])\n", " qso_df.at[i,\"accumulated_time_in_seat_by_band\"] = accumulated_time_in_seat\n", " \n", "for band in bands:\n", " set_time_in_seat_by_band(band)\n", "\n", "def points_missing_if_band_tail_missing(band:str, tail_duration) -> int:\n", " total_time_in_seat = g_by_band.get_group(band).iloc[-1][\"accumulated_time_in_seat_by_band\"]\n", " tail = g_by_band.get_group(band)[\n", " total_time_in_seat-tail_duration <= g_by_band.get_group(band)[\"accumulated_time_in_seat_by_band\"]]\n", " head = g_by_band.get_group(band)[\n", " total_time_in_seat-tail_duration > g_by_band.get_group(band)[\"accumulated_time_in_seat_by_band\"]]\n", "\n", " dxcc_count_tail_missing = len(g_by_band.get_group(band).groupby(\"DXCC\")) - len(head.groupby(\"DXCC\"))\n", " qso_points_tail = reduce(lambda x, y: int(x) + int(y), tail[\"QSO_POINTS\"])\n", " points_without_tail = (num_of_dxcc_bands - dxcc_count_tail_missing) * (qso_points - qso_points_tail)\n", " return claimed_score - points_without_tail\n", "\n", "\n", "durations = [600, 900, 1200, 1800, 3600]\n", "duration_keys = [f\"{tail_duration//60}'\" for tail_duration in durations]\n", "\n", "def tail_df_data():\n", " tail_dict = {}\n", " for tail_duration in durations:\n", " tail_dict[f\"{tail_duration//60}'\"] = \\\n", " [\n", " points_missing_if_band_tail_missing(band, tail_duration)\n", " for band in bands\n", " ]\n", " return tail_dict \n", " \n", "points_missing_if_band_tail_missing_df = pd.DataFrame(tail_df_data(), index=bands)\n", "points_missing_if_band_tail_missing_df.plot(\n", " y=duration_keys,\n", " kind=\"bar\",\n", " figsize=(7,12),\n", " title=f\"DL0ABT/p CWFD {year}, resultpoints missing if band tail missing\",\n", " subplots=True\n", ")\n", "\n", "print(points_missing_if_band_tail_missing_df)" ] }, { "cell_type": "code", "execution_count": null, "id": "b387b332-ed99-419f-8080-3ddafe85ba87", "metadata": {}, "outputs": [], "source": [ "qsos_per_operator = Counter()\n", "for qso in qsos:\n", " qsos_per_operator[qso[\"OPERATOR\"]] += 1\n", "qsos_per_operator" ] }, { "cell_type": "code", "execution_count": null, "id": "aa408f9a-6e36-400c-a865-e5ee745cded8", "metadata": {}, "outputs": [], "source": [ "def durations_on_band(some_qsos):\n", " \"\"\"On which band did the some station stay for how long?\"\"\"\n", " durations = []\n", " current_band = None\n", " start_of_current_band = None\n", " previous_qso_time = None\n", " for qso in some_qsos:\n", " band = qso[\"BAND\"]\n", " t = adif_io.time_on(qso)\n", " if current_band != band or 15*60 < (t-previous_qso_time).total_seconds():\n", " if start_of_current_band is not None and previous_qso_time is not None:\n", " delta_t = (previous_qso_time - start_of_current_band)\n", " durations.append((start_of_current_band, delta_t, current_band))\n", " current_band = band\n", " start_of_current_band = t\n", " previous_qso_time = t\n", " durations.append((start_of_current_band, t-start_of_current_band, current_band))\n", " return durations\n", "\n", "durations = durations_on_band(qsos)\n", "\n", "[f\"{t_dur_band[0].strftime('%H:%M:%S')} - {(t_dur_band[0]+t_dur_band[1]).strftime('%H:%M:%S')}\"\n", " f\", staying{t_dur_band[1].total_seconds()/60:7.1f} minutes on {t_dur_band[2]}\" \\\n", " for t_dur_band in durations]" ] }, { "cell_type": "code", "execution_count": null, "id": "db055551-094f-43ea-b288-80cb018d44cb", "metadata": {}, "outputs": [], "source": [ "def measure_minutes_between_qsos():\n", " minutes_between_qsos = Counter()\n", " last_qso_time = start_of_contest\n", " for qso in qsos:\n", " this_qso_time = adif_io.time_on(qso)\n", " time_between_qsos = int((this_qso_time - last_qso_time).total_seconds() / 60 + 0.5)\n", " minutes_between_qsos[time_between_qsos] += 1\n", " last_qso_time = this_qso_time\n", " max_time = max(minutes_between_qsos.keys())\n", " times = range(0, max_time+1)\n", " return pd.DataFrame(\n", " {\"time_between_qsos\": [minutes_between_qsos[t] for t in times]},\n", " index = times\n", " )\n", "gap_df = measure_minutes_between_qsos()\n", "gap_df.plot.bar(\n", " title=\"Gaps between adjacent QSOs\",\n", " figsize=(12,8),\n", " xlabel=\"minutes between QSOs\",\n", " ylabel=\"number of QSOs with no preceeding QSO within that many minutes\"\n", ")\n", "gap_df" ] }, { "cell_type": "markdown", "id": "14b7edb8-4629-4804-a74b-e34e0ea932b8", "metadata": {}, "source": [ "## FOM = \"Gummipunkte\"\n", "\n", "Die FOM-Zahlen versuchen, dem QSO einen Wert zuzuweisen.\n", "\n", "Der Grundgedanke ist: Wie viele Punkte würden insgesamt fehlen, wenn das QSO nicht im Log wäre?\n", "\n", "Das führte allerdings dazu, dass für jeden Multi nur das erste QSO, dass den Multi auf dem betreffenden Band\n", "arbeitet, die zusätzliche Punktzahl für den Multi bekäme und aller späteren QSOs diesbezüglich leer ausgingen.\n", "\n", "Daher wird der zusätzliche Gewinn für jeden Multi gleichmäßig auf alle QSOs verteilt,\n", "die den Multi geliefert haben. Ein Allerweltsmulti (Deutschland auf 80 m) liefert so\n", "jedem QSO nur einen geringen Beitrag, dass ihn liefert. Die zusätzlichen Punkte für\n", "einen Multi, den nur zwei QSOs geliefert haben, werden halbiert und auf die beiden QSOs verteilt." ] }, { "cell_type": "code", "execution_count": null, "id": "b0b30b86-22ce-4595-a269-6f97edc562f5", "metadata": {}, "outputs": [], "source": [ "def fom(qso) -> float:\n", " \"\"\"Figure of merrit - what do we think this QSO is worth?\"\"\"\n", " qso_points = int(qso[\"QSO_POINTS\"])\n", " if qso_points == 0:\n", " # Dupe QSO\n", " return 0\n", " else:\n", " multis = 1 / multi_count[multi(qso)]\n", " # return (TOTAL_QSO_POINTS * TOTAL_MULTIS - (TOTAL_QSO_POINTS - qso_points) * (TOTAL_MULTIS - multis)) / TOTAL_MULTIS\n", " # return (TOTAL_QSO_POINTS * TOTAL_MULTIS - \\\n", " # (TOTAL_QSO_POINTS * TOTAL_MULTIS - TOTAL_QSO_POINTS * multis - qso_points * TOTAL_MULTIS + qso_points * multis)) / TOTAL_MULTIS\n", " # return (TOTAL_QSO_POINTS * multis + qso_points * TOTAL_MULTIS - qso_points * multis) / TOTAL_MULTIES\n", " return qso_points * (1 - multis / total_multis) + multis * total_qso_points / total_multis\n", " \n", "for qso in qsos:\n", " qso[\"FOM\"] = fom(qso)\n", "\n", "# Redefine the DF with FOM:\n", "qso_df = pd.DataFrame(qsos)" ] }, { "cell_type": "code", "execution_count": null, "id": "1d1a2193-e027-44b1-8704-10b87ff4b0af", "metadata": {}, "outputs": [], "source": [ "# I want these to be sorted by wavelength, so convert to int, sort,\n", "# and convert back to string with trailing \"M\".\n", "bands = [f\"{b}M\" for b in sorted(set((int(qso[\"BAND\"][0:-1]) for qso in qsos)))]\n", "g_by_band = qso_df.groupby(qso_df[\"BAND\"])" ] }, { "cell_type": "code", "execution_count": null, "id": "f042bf98-8a06-4680-8e89-5f8afb5c7282", "metadata": {}, "outputs": [], "source": [ "# Just for the fun of it, list the few QSOs we have on 10 m:\n", "# It demonstrates how FOM work:\n", "g_by_band.get_group(\"10M\")[[\"CALL\", \"TIME_ON\", \"DXCC_NAME\", \"MODE\", \"OPERATOR\", \\\n", " \"QSO_POINTS\", \"FOM\", \"APP_N1MM_ISRUNQSO\"]]" ] }, { "cell_type": "code", "execution_count": null, "id": "63ac6b77-d081-4260-bd99-2fd0485add37", "metadata": {}, "outputs": [], "source": [ "from datetime import timedelta\n", "from functools import reduce\n", "def qso_points_for_band(band: str) -> int:\n", " return reduce(lambda x, y: x + int(y), g_by_band.get_group(band)[\"QSO_POINTS\"], 0)\n", "\n", "band_df = pd.DataFrame(\n", " {\n", " \"QSO Count\": [len(g_by_band.get_group(band)) for band in bands],\n", " \"QSO Points\": [qso_points_for_band(band) for band in bands]\n", " },\n", " index=bands\n", ")\n", "\n", "def fom_for_band(band: str) -> float:\n", " return reduce(lambda x, y: x + float(y), g_by_band.get_group(band)[\"FOM\"], 0)\n", "\n", "points_per_qso_df = pd.DataFrame(\n", " {\"Avg. points per QSO\": [\n", " band_df.loc[band][\"QSO Points\"] / band_df.loc[band][\"QSO Count\"]\n", " for band in bands\n", " ],\n", " \"Avg. FOM per QSO\": [\n", " fom_for_band(band) / band_df.loc[band][\"QSO Count\"]\n", " for band in bands \n", " ]\n", " },\n", " index=bands\n", ")\n", "points_per_qso_df\n", "fig, axs = plt.subplots(1,2,figsize=(12, 7))\n", "points_per_qso_df.plot(kind=\"bar\", ax=axs, figsize=(7,7), title=f\"{station_callsign} {CONTEST_ID} {year}, \"\n", " f\"avg. QSO points per QSO\", legend=False, subplots=True, use_index=True)\n", "points_per_qso_df.round(1)" ] }, { "cell_type": "code", "execution_count": null, "id": "8e059caf-96ff-4fea-aa23-39bd179c0e9f", "metadata": {}, "outputs": [], "source": [ "from typing import Callable\n", "from datetime import datetime\n", "import heapq\n", "\n", "def something_per_hour(qsos: list[qso], window_halfwidth: timedelta, interesting_moments: list[datetime], qso_to_float: Callable[[qso], float]) -> list[float]:\n", " window_halfwidth_seconds = window_halfwidth.total_seconds()\n", " window_scale_factor = 3600 / window_halfwidth_seconds\n", " result : list[float] = []\n", " time_and_i_and_qsos = [(adif_io.time_on(qso), i, qso) for qso, i in zip(qsos, range(0,len(qsos)))]\n", " relevant_time_and_i_and_qsos = []\n", " k = 0\n", " for interesting_moment in interesting_moments:\n", " start_interesting_interval = interesting_moment - window_halfwidth\n", " end_interesting_interval = interesting_moment + window_halfwidth\n", " while 0 < len(relevant_time_and_i_and_qsos) \\\n", " and relevant_time_and_i_and_qsos[0][0] <= start_interesting_interval:\n", " heapq.heappop(relevant_time_and_i_and_qsos)\n", " if k < len(time_and_i_and_qsos) and time_and_i_and_qsos[k][0] < end_interesting_interval:\n", " heapq.heappush(relevant_time_and_i_and_qsos, time_and_i_and_qsos[k])\n", " k += 1\n", " sum = 0\n", " for time, _, qso in relevant_time_and_i_and_qsos:\n", " sum += qso_to_float(qso) * window_scale_factor * \\\n", " (1 - abs((interesting_moment-time).total_seconds()) / window_halfwidth_seconds)\n", " result.append(sum)\n", " return result\n", "\n", "def get_interesting_moments(qsos: list[qso], window_halfwidth: timedelta) -> list[datetime]:\n", " result_raw: list[datetime] = [start_of_contest-window_halfwidth, end_of_contest+window_halfwidth]\n", " for qso in qsos:\n", " qso_t = adif_io.time_on(qso)\n", " result_raw += [qso_t-window_halfwidth, qso_t, qso_t + window_halfwidth]\n", " result_with_dupes = sorted(result_raw)\n", " result = []\n", " last_t = result_with_dupes[0]\n", " result.append(last_t)\n", " for t in result_with_dupes:\n", " if last_t < t:\n", " result.append(t)\n", " last_t = t\n", " return result\n", "\n", "\n", "def get_fom_df(window_halfwidth: timedelta):\n", "\n", " interesting_moments = get_interesting_moments(qsos, window_halfwidth)\n", "\n", " return pd.DataFrame(\n", " {\n", " \"QSOs per hour\": something_per_hour(qsos, window_halfwidth,interesting_moments, lambda _: 1.0),\n", " \"QSO points per hour\": something_per_hour(qsos, window_halfwidth, interesting_moments, lambda qso: float(qso[\"QSO_POINTS\"])),\n", " \"FOM per_hour\": something_per_hour(qsos, window_halfwidth,interesting_moments, lambda qso: float(qso[\"FOM\"])),\n", " \n", " },\n", " index=interesting_moments\n", " )\n", "\n", "start = datetime.now()\n", "various_per_hour = get_fom_df(timedelta(minutes=15))\n", "done = datetime.now()\n", "print(f\"That took {(done-start).total_seconds()} s\")\n", "\n", "various_per_hour[[\"QSOs per hour\"]].plot(\n", " title=f\"{station_callsign} {CONTEST_ID} {year}, qsos per hour rates\",\n", " figsize=(12,7), grid=True\n", ")\n", "\n", "various_per_hour.plot(\n", " title=f\"{station_callsign} {CONTEST_ID} {year}, per hour rates\",\n", " figsize=(12,7), grid=True\n", ")\n", "\n", "\n", "pass" ] }, { "cell_type": "code", "execution_count": null, "id": "3fdaad1f-0142-40b3-8450-5c38a3dfc084", "metadata": {}, "outputs": [], "source": [ "def get_op_df(op: str, window_halfwidth: timedelta):\n", "\n", " qsos_filtered = [qso for qso in qsos if qso[\"OPERATOR\"] == op]\n", "\n", " interesting_moments = get_interesting_moments(qsos_filtered, window_halfwidth)\n", "\n", " return pd.DataFrame(\n", " {\n", " \"FOM per_hour\": something_per_hour(qsos_filtered, window_halfwidth,interesting_moments, lambda qso: float(qso[\"FOM\"])),\n", " \"QSO points per hour\": something_per_hour(qsos_filtered, window_halfwidth, interesting_moments, lambda qso: float(qso[\"QSO_POINTS\"])),\n", " \"QSOs per hour\": something_per_hour(qsos_filtered, window_halfwidth,interesting_moments, lambda _: 1.0),\n", " \n", " },\n", " index=interesting_moments\n", " )\n", "\n", "for operator in sorted(qsos_per_operator.keys()):\n", "\n", " op_per_hour = get_op_df(operator, timedelta(minutes=15))\n", "\n", " op_per_hour[[\"QSOs per hour\"]].plot(\n", " title=f\"{station_callsign} {CONTEST_ID} {year}, OP {operator} qsos per hour rates\",\n", " figsize=(12,7), grid=True\n", " )\n", "\n", " op_per_hour.plot(\n", " title=f\"{station_callsign} {CONTEST_ID} {year}, OP {operator} per hour rates\",\n", " figsize=(12,7), grid=True\n", " )\n" ] }, { "cell_type": "code", "execution_count": null, "id": "ddd5df2e-9f26-4fc4-9fde-1d62cd279a3b", "metadata": {}, "outputs": [], "source": [ "def block_fom(qsos) -> float:\n", " result = 0.0\n", " for qso in qsos:\n", " if \"FOM\" in qso:\n", " result += float(qso[\"FOM\"])\n", " else:\n", " raise RuntimeError(f\"Key fom missing in: {qso}\")\n", " return result\n", "\n", "def block_label(qso) -> str:\n", " rsp = \"RUN\" if \"1\" == qso[\"APP_N1MM_ISRUNQSO\"] else \"S+P\"\n", " # return f\" {qso['OPERATOR']}: {qso['BAND']} {qso['MODE']} {rsp}\"\n", " return f\" {qso['OPERATOR']}: {qso['BAND']} {rsp}\"\n", "\n", "# def get_start_of_contest() -> datetime:\n", "# return start_of_contest;\n", "\n", "def duration(block_start, raw_block) -> float:\n", " return (adif_io.time_on(raw_block[-1]) - block_start).total_seconds()\n", "\n", "def generate_block_df(some_qsos: list[qso]) -> pd.DataFrame:\n", " current_block_start = adif_io.time_on(some_qsos[0])\n", " current_block_label = block_label(some_qsos[0])\n", " current_block = []\n", " raw_blocks = []\n", " for qso in some_qsos:\n", " this_block_label = block_label(qso)\n", " if this_block_label == current_block_label:\n", " current_block.append(qso)\n", " elif 1 <= len(current_block):\n", " # current_block is complete:\n", " raw_blocks.append(\n", " (current_block_label,\n", " current_block_start,\n", " current_block,\n", " duration(current_block_start, current_block),)\n", " )\n", " current_block_start = adif_io.time_on(qso)\n", " current_block = [qso]\n", " current_block_label = this_block_label\n", " # Don't forget the last block:\n", " if 1 <= len(current_block):\n", " raw_blocks.append((current_block_label, current_block_start, current_block, duration(current_block_start, current_block), ))\n", " return pd.DataFrame(\n", " {\n", " \"label\": [rb[0] for rb in raw_blocks],\n", " \"start\": [rb[1].strftime(\"%H:%M:%S\") for rb in raw_blocks],\n", " \"end\": [(rb[1]+timedelta(seconds=rb[3])).strftime(\"%H:%M:%S\") for rb in raw_blocks],\n", " \"duration_minutes\": [rb[3]/60 for rb in raw_blocks],\n", " \"fom_per_hour\": [block_fom(rb[2]) / rb[3] * 3600 if rb[3] > 0.0 else None for rb in raw_blocks],\n", " \"qsos\": [len(rb[2]) for rb in raw_blocks],\n", " \"qsos_per_hour\": [len(rb[2]) / rb[3] * 3600 if rb[3] > 0.0 else None for rb in raw_blocks],\n", " }\n", " )\n", "main_blocks = generate_block_df(qsos)\n", "main_blocks[[\"label\", \"start\", \"end\", \"duration_minutes\", \"qsos\", \"qsos_per_hour\", \"fom_per_hour\"]].round(1)" ] }, { "cell_type": "code", "execution_count": null, "id": "30b31e0b-5739-4acc-afcd-3d6fb2da6b80", "metadata": {}, "outputs": [], "source": [ "# FOM größerer Aktionen (länger als 10 Minuten an derselben Sache, ohne Pause),\n", "# nach FOM-Erfolg sortiert:\n", "main_blocks[10 <= main_blocks[\"duration_minutes\"]].sort_values(\"fom_per_hour\", ascending=False)\\\n", " [[\"label\", \"start\", \"end\", \"duration_minutes\",\"qsos\", \"qsos_per_hour\", \"fom_per_hour\"]].round(1)" ] }, { "cell_type": "code", "execution_count": null, "id": "663405fc-fae9-40a3-b7e2-001ab9d87b2e", "metadata": {}, "outputs": [], "source": [ "# Correlation between QSOs / hour and FOM / hour?\n", "qso_rate_to_fph = main_blocks[10 <= main_blocks[\"duration_minutes\"]].sort_values(\"qsos_per_hour\", ascending=True)\\\n", "[[\"label\", \"qsos_per_hour\", \"fom_per_hour\"]]\n", "\n", "correlation_df = pd.DataFrame(\n", " {\n", " \"fom per hour\": [row[\"fom_per_hour\"]for i, row in qso_rate_to_fph.iterrows()]\n", " },\n", " index = [row[\"qsos_per_hour\"] for i, row in qso_rate_to_fph.iterrows()]\n", ")\n", "\n", "correlation_df.plot(\n", " figsize=(10,6),\n", " xlabel=\"QSOs per hour\"\n", ")\n", "\n", "qso_rate_to_fph.round(1)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.5" } }, "nbformat": 4, "nbformat_minor": 5 }