427 lines
20 KiB
Python
427 lines
20 KiB
Python
#
|
|
# This script generates a weekly report from our DMR Net Check-In data.
|
|
#
|
|
#
|
|
|
|
import pandas as pd
|
|
#import odf
|
|
import os
|
|
import sys
|
|
#import logzero
|
|
|
|
|
|
# setup output parameters for jupyter interactive execution
|
|
# Use 3 decimal places in output display
|
|
pd.set_option("display.precision", 0)
|
|
|
|
# Don't wrap repr(DataFrame) across additional lines
|
|
pd.set_option("display.expand_frame_repr", False)
|
|
|
|
# Set max rows displayed in output to 25
|
|
pd.set_option("display.max_rows", 150)
|
|
|
|
|
|
# point to our various input files - adjust this to your environment
|
|
data_dir = '/home/ed/syncthing/ham-radio/ARES-RACES/Nets--PNW_ARES_DMR_Weekly_Net'
|
|
filename = 'Check_In_Data.xlsx'
|
|
data_file = os.path.join(data_dir, filename)
|
|
output_file = os.path.join(data_dir, "PNW_Digital_ARES_EMCOMM_Weekly_Net_Check_In_Form.txt")
|
|
|
|
# Our check-in form as a dictionary
|
|
blank_checkin_form_dict = {
|
|
'ID': { '1':{'Boundary':[],'Bonner':[],'Kootenai':[],'Benewah':[],'Shoshone':[]},
|
|
'2':{'Latah':[],'Nez Pierce':[],'Lewis':[],'Clearwater':[],'Idaho':[]},
|
|
'3':{'Adams':[],'Washington':[],'Payette':[],'Gem':[],'Canyon':[],'Ada':[],
|
|
'Owyhee':[],'Valley':[],'Boise':[],'Elmore':[]},
|
|
'4':{'Camas':[],'Blaine':[],'Gooding':[],'Lincoln':[],'Jerome':[],'Twin Falls':[],
|
|
'Minidoka':[],'Cassia':[]},
|
|
'5':{'Bingham':[],'Power':[],'Bannock':[],'Oneida':[],'Franklin':[],
|
|
'Bear Lake':[],'Caribou':[]},
|
|
'6':{'Lemhi':[],'Custer':[],'Butte':[],'Clark':[],'Jefferson':[],'Fremont':[],
|
|
'Madison':[],'Teton':[],'Bonneville':[]}
|
|
},
|
|
'OR': { '1':{'Clatsop':[],'Columbia':[],'Tillamook':[],'Washington':[],'Multnomah':[],
|
|
'Clackamas':[]},
|
|
'2':{'Hood River':[],'Wasco':[],'Sherman':[],'Jefferson':[],'Deschutes':[],
|
|
'Crook':[]},
|
|
'3':{'Gilliam':[],'Wheeler':[],'Morrow':[],'Umatilla':[],'Union':[],
|
|
'Wallowa':[]},
|
|
'4':{'Yamhill':[],'Polk':[],'Lincoln':[],'Benton':[],'Marion':[],
|
|
'Linn':[],'Lane':[]},
|
|
'5':{'Douglas':[],'Coos':[],'Curry':[],'Josephine':[],
|
|
'Jackson':[]},
|
|
'6':{'Klamath':[],'Lake':[],'Harney':[],'Grant':[],'Baker':[],
|
|
'Malheur':[]}
|
|
},
|
|
'WA': { '1':{'Island':[], 'San Juan':[], 'Skagit':[], 'Snohomish':[], 'Whatcom':[], 'unknown':[]},
|
|
'2':{'Clallam':[],'Jefferson':[],'Kitsap':[], 'unknown':[]},
|
|
'3':{'Grays Harbor':[],'Lewis':[],'Mason':[],'Pacific':[],'Thurston':[]},
|
|
'4':{'Clark':[],'Cowlitz':[],'Skamania':[],'Wahkiakum':[]},
|
|
'5':{'Pierce':[]},
|
|
'6':{'King':[]},
|
|
'7':{'Chelan':[],'Douglas':[],'Grant':[],'Kittitas':[],'Okanogan':[]},
|
|
'8':{'Benton':[],'Franklin':[],'Klickitat':[],'Walla Walla':[],'Yakima':[]},
|
|
'9':{'Adams':[],'Asotin':[],'Columbia':[],'Ferry':[],'Garfield':[],'Lincoln':[],
|
|
'Pend Orielle':[],'Spokane':[],'Stevens':[],'Whitman':[]},
|
|
'State EMD':{'none':[]}
|
|
},
|
|
'Philippines': { '-':{'-':[]}
|
|
},
|
|
'Canada': {'n/a':{'n/a':[]}
|
|
},
|
|
'Visitor' : { '-':{'-':[]}
|
|
}
|
|
}
|
|
|
|
|
|
# create dataframes from input file
|
|
print("Reading input file from: ")
|
|
print(" ", data_file)
|
|
xls = pd.ExcelFile(data_file, engine='openpyxl')
|
|
checkins_df = pd.read_excel(xls, "Check-ins", dtype=str)
|
|
checkins_df["Date"] = pd.to_datetime(checkins_df["Date"]).dt.strftime('%Y-%m-%d')
|
|
#checkins_df['Date'] = checkins_df['Date'].strftime('%Y-%m-%d')
|
|
#print("checkins_df:")
|
|
#print(checkins_df)
|
|
callinfo_df = pd.read_excel(xls, "Call_Data", dtype=str)
|
|
#print("callinfo_df:")
|
|
#print(callinfo_df)
|
|
hh_dir_df = pd.read_excel(xls, "Hamshack_Hotline", dtype=str)
|
|
#print("hh_dir_df:")
|
|
#print(hh_dir_df)
|
|
|
|
# build dictionary of HH phone numbers
|
|
hh_phone_dict = {"Callsign" : "HH Number"}
|
|
for i,row in hh_dir_df.iterrows():
|
|
callsign = row[1]
|
|
hh_num = row[7]
|
|
# we only add the hh number first seen for a unique callsign
|
|
if callsign not in hh_phone_dict.keys():
|
|
hh_phone_dict.update({callsign : hh_num})
|
|
|
|
# fill in any missing data (NAN's) with default text
|
|
callinfo_df['Name'].fillna('', inplace=True)
|
|
callinfo_df['State'].fillna('none', inplace=True)
|
|
callinfo_df['District'].fillna('unknown', inplace=True)
|
|
callinfo_df['County'].fillna('unknown', inplace=True)
|
|
callinfo_df['Affiliation'].fillna('', inplace=True)
|
|
|
|
# build dictionary of calls indexed by check-in date
|
|
calls_on_date_dict = {}
|
|
checkin_count_dict = {"Callsign" : "Check-in Count"}
|
|
for i,row in checkins_df.iterrows():
|
|
checkin_date = row[0]
|
|
checkin_call = row[1]
|
|
if checkin_date in calls_on_date_dict.keys():
|
|
if checkin_call not in calls_on_date_dict[checkin_date]:
|
|
calls_on_date_dict[checkin_date].append(checkin_call)
|
|
else:
|
|
calls_on_date_dict.update({checkin_date:[checkin_call]})
|
|
if checkin_call in checkin_count_dict.keys():
|
|
checkin_count_dict[checkin_call] = checkin_count_dict[checkin_call] + 1
|
|
else:
|
|
checkin_count_dict.update({checkin_call : 1})
|
|
|
|
# build dictionary of call info indexed by callsign
|
|
call_data_dict = {}
|
|
for i,row in callinfo_df.iterrows():
|
|
# row of data is call, name, state, district, county, affiliation, hh_num
|
|
# dictionary becomes: { call: [name[0], state[1], district[2], county[3], affiliation[4]], hh_num[5]}
|
|
callsign = row[0]
|
|
if callsign in hh_phone_dict.keys():
|
|
hh_num = hh_phone_dict[callsign]
|
|
else:
|
|
hh_num = ""
|
|
call_data_dict.update({callsign:[row[1],row[2],row[3],row[4],row[5],hh_num]})
|
|
|
|
# build checkin_form_dict dictionary
|
|
checkin_form_dict = blank_checkin_form_dict
|
|
for call in call_data_dict.keys():
|
|
#print("Looking at:", call)
|
|
#if call in hh_phone_dict.keys():
|
|
#print("Call "+call+" has HH VOIP #"+hh_phone_dict[call])
|
|
call_state = call_data_dict[call][1]
|
|
call_dist = call_data_dict[call][2]
|
|
call_county = call_data_dict[call][3]
|
|
call_affil = call_data_dict[call][4]
|
|
call_hh_num = call_data_dict[call][5]
|
|
#print("State, Dist, Cnty, Affil, HH Num: ", call_state, call_dist, call_county, call_affil, call_hh_num)
|
|
|
|
# sanity check the data - error if not found in dictionary keys...
|
|
# print("Verifying at: ", call, call_state, call_dist, call_county)
|
|
if call_state not in checkin_form_dict.keys():
|
|
print("ERROR: ",call_state,"not in checkin_form_dict!")
|
|
if call_dist not in checkin_form_dict[call_state].keys():
|
|
print("ERROR: ",call_dist, "not in checkin_form_dict under state of", call_state,"!")
|
|
if call_county not in checkin_form_dict[call_state][call_dist].keys():
|
|
print("ERROR: ",call_county, "not in checkin_form_dict under district ", call_dist,"!")
|
|
|
|
# okay, safe to move on...
|
|
call_list = checkin_form_dict[call_state][call_dist][call_county]
|
|
#print(" calls in",call_county,":", call_list)
|
|
if call not in call_list:
|
|
checkin_form_dict[call_state][call_dist][call_county].append(call)
|
|
|
|
|
|
|
|
# print check-in form based on historical check-ins
|
|
with open(output_file,"w") as outfile:
|
|
outfile.write("\n")
|
|
outfile.write("PNW Digital ARES & EMCOMM Check-In Net - Check-ins for _______________\n\n")
|
|
for state in checkin_form_dict.keys():
|
|
if state not in ['Philippines','Canada','Visitor']:
|
|
outfile.write("State: " + state +"\n")
|
|
for district in checkin_form_dict[state].keys():
|
|
outfile.write(" District: " + district + "\n")
|
|
if district not in ["State EMD"]:
|
|
|
|
# print the list of counties in this district
|
|
outfile.write(" Counties: ")
|
|
county_list = sorted(checkin_form_dict[state][district])
|
|
county_count = len(county_list)
|
|
index = 0
|
|
if county_count >= 1:
|
|
outfile.write(county_list[index])
|
|
county_count -= 1
|
|
index += 1
|
|
while county_count >= 1:
|
|
if county_list[index] == "unknown":
|
|
county_count -= 1
|
|
index += 1
|
|
continue
|
|
|
|
if (index % 5) == 0:
|
|
outfile.write("\n "+county_list[index])
|
|
else:
|
|
outfile.write(", "+county_list[index])
|
|
county_count -= 1
|
|
index += 1
|
|
outfile.write("\n\n")
|
|
|
|
# now collect all of the calls from all counties
|
|
call_list = []
|
|
for county in sorted(checkin_form_dict[state][district]):
|
|
for county_call_item in checkin_form_dict[state][district][county]:
|
|
call_list.append(county_call_item)
|
|
call_list.sort()
|
|
for call in call_list:
|
|
call_name = call_data_dict[call][0]
|
|
call_county = call_data_dict[call][3]
|
|
call_affil = call_data_dict[call][4]
|
|
if call in checkin_count_dict.keys():
|
|
checkin_count = checkin_count_dict[call]
|
|
checkin_str = ", [{}]".format(checkin_count)
|
|
else:
|
|
checkin_count = 0
|
|
checkin_str = ""
|
|
if checkin_count > 0:
|
|
outfile.write(" "+call+", "+call_name+checkin_str)
|
|
if call in hh_phone_dict.keys():
|
|
outfile.write(", HH VOIP #"+hh_phone_dict[call])
|
|
if call_affil != "":
|
|
outfile.write(", "+call_affil+"\n")
|
|
else:
|
|
outfile.write("\n")
|
|
else:
|
|
for county in checkin_form_dict[state][district].keys():
|
|
call_list = checkin_form_dict[state][district][county]
|
|
call_list.sort()
|
|
for call in call_list:
|
|
if call in checkin_count_dict.keys():
|
|
checkin_count = checkin_count_dict[call]
|
|
checkin_str = ", [{}]".format(checkin_count)
|
|
else:
|
|
checkin_count = 0
|
|
checkin_str = ""
|
|
if checkin_count > 0:
|
|
outfile.write(" "+call+", "+call_data_dict[call][0]+checkin_str)
|
|
if call in hh_phone_dict.keys():
|
|
outfile.write(", HH VOIP #"+hh_phone_dict[call])
|
|
affil = call_data_dict[call][4]
|
|
if affil != '':
|
|
outfile.write(", "+affil+"\n")
|
|
else:
|
|
outfile.write("\n")
|
|
outfile.write("\n")
|
|
outfile.write("\n")
|
|
else:
|
|
if state == "Visitor":
|
|
outfile.write("Visitor Check-ins:\n")
|
|
else:
|
|
outfile.write("Country: "+state+"\n")
|
|
for district in checkin_form_dict[state].keys():
|
|
for county in checkin_form_dict[state][district].keys():
|
|
call_list = checkin_form_dict[state][district][county]
|
|
call_list.sort()
|
|
for call in call_list:
|
|
if call in checkin_count_dict.keys():
|
|
checkin_count = checkin_count_dict[call]
|
|
checkin_str = ", [{}]".format(checkin_count)
|
|
else:
|
|
checkin_count = 0
|
|
checkin_str = ""
|
|
outfile.write(" "+call+", "+call_data_dict[call][0]+checkin_str)
|
|
if call in hh_phone_dict.keys():
|
|
outfile.write(", HH VOIP #"+hh_phone_dict[call])
|
|
affil = call_data_dict[call][4]
|
|
if affil != '':
|
|
outfile.write(", "+affil+"\n")
|
|
else:
|
|
outfile.write("\n")
|
|
outfile.write("\n")
|
|
outfile.write("\n")
|
|
|
|
outfile.close()
|
|
|
|
|
|
# generate net reports based on check-in data
|
|
report_dir = os.path.join(data_dir, "reports")
|
|
net_day_list = checkins_df['Date'].unique()
|
|
#print("DEBUG: net_dat_list =", net_day_list)
|
|
for net_day in net_day_list:
|
|
|
|
#print("Looking at checkins for: ", net_day)
|
|
|
|
# get list of calls checked in on this day
|
|
call_list = calls_on_date_dict[net_day]
|
|
num_checkins = len(call_list)
|
|
#print(" ",num_checkins,"Check-ins on this day: ",call_list)
|
|
|
|
# fill in report dictionary from call list
|
|
report_dict = {}
|
|
for call in call_list:
|
|
|
|
if call not in call_data_dict.keys():
|
|
print("Unknown call in call database: {}".format(call))
|
|
continue
|
|
|
|
#print("report_dict:")
|
|
#print(report_dict)
|
|
#print("")
|
|
#print("Looking at:", call)
|
|
call_state = call_data_dict[call][1]
|
|
call_dist = call_data_dict[call][2]
|
|
call_county = call_data_dict[call][3]
|
|
call_affil = call_data_dict[call][4]
|
|
#print(" State, Dist, Cnty, Affil: ", call_state, call_dist, call_county, call_affil)
|
|
|
|
# sanity check the data - error if not found in dictionary keys...
|
|
#print("Verifying at: ", call, call_state, call_dist, call_county)
|
|
if call_state not in report_dict.keys():
|
|
# make new state entry
|
|
report_dict.update({call_state:{call_dist:{call_county:[call]}}})
|
|
#print("ERROR: ",call_state,"not in report_dict!")
|
|
continue
|
|
if call_dist not in report_dict[call_state].keys():
|
|
report_dict[call_state].update({call_dist:{call_county:[call]}})
|
|
#print("ERROR: ",call_dist, "not in report_dict under state of", call_state,"!")
|
|
continue
|
|
if call_county not in report_dict[call_state][call_dist].keys():
|
|
report_dict[call_state][call_dist].update({call_county:[call]})
|
|
#print("ERROR: ",call_county, "not in report_dict under district ", call_dist,"!")
|
|
continue
|
|
|
|
# okay, safe to move on...
|
|
report_call_list = report_dict[call_state][call_dist][call_county]
|
|
#print(" calls in",call_county,":", call_list)
|
|
if call not in report_call_list:
|
|
report_dict[call_state][call_dist][call_county].append(call)
|
|
|
|
# Now print a weekly report for this net_day
|
|
|
|
output_file = os.path.join(report_dir, "Weekly_Check-in_Report_for_"+net_day+".txt")
|
|
#print("Weekly report for",net_day,"will be written here:")
|
|
#print(" "+output_file)
|
|
with open(output_file,"w") as outfile:
|
|
|
|
# Opening text
|
|
outfile.write("\n ")
|
|
outfile.write("PNW Digital ARES & EMCOMM Check-In Net - "+str(num_checkins)+" check-ins on "+net_day+"\n")
|
|
outfile.write("\n")
|
|
outfile.write("We had a total of "+str(num_checkins)+" check-ins on "+net_day+" to the Pacific Northwest (PNW)\n")
|
|
outfile.write("Digital ARES & EMCOMM Check-In Net. Below is the detailed check-in list grouped\n")
|
|
outfile.write("by ARRL ARES districts. If any of the info (i.e. name or agency affiliation) below is\n")
|
|
outfile.write("incomplete or incorrect, please call me on Hamshack Hotline (HH) VOIP x11893, or\n")
|
|
outfile.write("e-mail \"ed@n7ekb.net\" with the correction(s).\n\n")
|
|
|
|
# Check-in details
|
|
for state in sorted(report_dict):
|
|
if state not in ['Philippines','Canada','Visitor']:
|
|
outfile.write(state+" State:\n")
|
|
for district in sorted(report_dict[state]):
|
|
if district not in ["State EMD"]:
|
|
outfile.write(" District: " + district + "\n")
|
|
for county in sorted(report_dict[state][district]):
|
|
cur_call_list = report_dict[state][district][county]
|
|
if len(cur_call_list) > 0:
|
|
outfile.write(" " + county + " County:\n")
|
|
cur_call_list.sort()
|
|
for call in cur_call_list:
|
|
affil = call_data_dict[call][4]
|
|
hh_num = call_data_dict[call][5]
|
|
outfile.write(" "+call+", "+call_data_dict[call][0])
|
|
if hh_num != '':
|
|
outfile.write(", HH VOIP #"+hh_num)
|
|
if affil != '':
|
|
outfile.write(", "+affil+"\n")
|
|
else:
|
|
outfile.write("\n")
|
|
else:
|
|
outfile.write(" "+district+"\n")
|
|
for county in sorted(report_dict[state][district]):
|
|
cur_call_list = report_dict[state][district][county]
|
|
cur_call_list.sort()
|
|
for call in cur_call_list:
|
|
affil = call_data_dict[call][4]
|
|
hh_num = call_data_dict[call][5]
|
|
outfile.write(" "+call+", "+call_data_dict[call][0])
|
|
if hh_num != '':
|
|
outfile.write(", HH VOIP #"+hh_num)
|
|
if affil != '':
|
|
outfile.write(", "+affil+"\n")
|
|
else:
|
|
outfile.write("\n")
|
|
#outfile.write("\n")
|
|
outfile.write("\n")
|
|
for state in sorted(report_dict):
|
|
if state in ['Canada','Philippines','Visitor']:
|
|
outfile.write(state+":\n")
|
|
for district in sorted(report_dict[state]):
|
|
for county in sorted(report_dict[state][district]):
|
|
cur_call_list = report_dict[state][district][county]
|
|
cur_call_list.sort()
|
|
for call in cur_call_list:
|
|
affil = call_data_dict[call][4]
|
|
hh_num = call_data_dict[call][5]
|
|
outfile.write(" "+call+", "+call_data_dict[call][0])
|
|
if hh_num != '':
|
|
outfile.write(", HH VOIP #"+hh_num)
|
|
if affil != '':
|
|
outfile.write(", "+affil+"\n")
|
|
else:
|
|
outfile.write("\n")
|
|
outfile.write("\n")
|
|
outfile.write("\n")
|
|
|
|
# output the footer text...
|
|
outfile.write("\nAbout the net:\n\n")
|
|
outfile.write("The PNW Digital ARES & EMCOMM Check-In Net is held every Saturday evening at 7:30 PM local\n")
|
|
outfile.write("time on PNW Regional, DMR talk group 31771 (available on both the PNW Digital http://pnwdigital.net\n")
|
|
outfile.write("and Brandmeister DMR networks). Anyone interested in Amateur Radio Emergency Communications is\n")
|
|
outfile.write("welcome to check-in. The net is an opportunity for DMR-capable ARES and EMCOMM hams to exercise\n")
|
|
outfile.write("their DMR equipment in a regional directed net. The net demonstrates the wide coverage area and\n")
|
|
outfile.write("capability of DMR repeaters and hot spots in our Pacific Northwest region. It also highlights the\n")
|
|
outfile.write("wide range of EMCOMM-related organizations who have members with DMR capability.\n\n")
|
|
|
|
# close this week's report file
|
|
outfile.close()
|
|
|
|
print("Processed {} check-ins for {}.".format(num_checkins, net_day))
|
|
print("All Done!")
|
|
|
|
|
|
# In[ ]:
|
|
|
|
|
|
|
|
|