378 lines
12 KiB
Plaintext
378 lines
12 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "55bc224f879f844a",
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import xml.etree.ElementTree\n",
|
|
"import xml.etree.ElementTree as ET\n",
|
|
"import numpy as np\n",
|
|
"import pandas as pd"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"outputs": [],
|
|
"source": [
|
|
"DEVICES_LAB = {\n",
|
|
" # 'a8:03:2a:b1:35:60': 'dev01-shelly-plus-1pm-relais', # Christoph\n",
|
|
" # '60:01:94:c7:69:ac': 'dev02-sonoff-relais', # Oliver\n",
|
|
" # 'd8:f1:5b:d8:08:0c': 'dev03-tuya-lampe', # Leonard\n",
|
|
" # '1c:d6:bd:b5:d3:bb': 'dev04-linkind-zigbee-mini-hub', # Thorsten\n",
|
|
" # '6c:5a:b0:7d:e2:25': 'dev05-tplink-tapo-l530e-birne', # Suad\n",
|
|
" # '54:af:97:7c:5e:f0': 'dev06-tapo-steckdose', # Andreas\n",
|
|
" # '74:ab:93:de:a0:7e': 'dev07-blink-sicherheitskamera', # Julian\n",
|
|
" '34:25:be:ef:91:bf': 'dev08-echo-dot-l4s3re', # Alexander\n",
|
|
" # '70:ee:50:90:64:04': 'dev09-netatmo-smart-weather-station', # Jakobus\n",
|
|
" # '68:3a:48:4b:53:c5': 'dev10-aeotec-z-wave-hub', # Andre\n",
|
|
" # 'dc:ed:83:4a:cf:76': 'dev11-aqara-presence-sensor-fp2', # Artur\n",
|
|
" # '8c:f6:81:dc:63:54': 'dev12-shelly-bewegungsmelder', # Moritz\n",
|
|
" # '24:4c:ab:43:0d:0f': 'dev13-shelly-flood', # Mohamad\n",
|
|
" # '08:b6:1f:cc:4d:c0': 'dev14-shelly-ht-temperatur-sensor', # Bastian\n",
|
|
" # '90:48:6c:17:ae:25': 'dev15-ring-door-camera' # Victor (Ring hinzugefügt, device Nummern angepasst)\n",
|
|
"}"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"id": "b5715a859aec218c"
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "39ad609a78e03fa0",
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"MINIMAL_FEATURES = [\n",
|
|
" #Meta\n",
|
|
" '_ws.malformed/_ws.expert/_ws.malformed.expert',\n",
|
|
" # 'bootp/bootp.dhcp'\n",
|
|
" # 'bootp/bootp.flags.bc'\n",
|
|
" # 'bootp/bootp.flags.reserved'\n",
|
|
" # 'bootp/bootp.hops'\n",
|
|
" # 'bootp/bootp.hw.len'\n",
|
|
" # 'bootp/bootp.hw.type'\n",
|
|
" # 'bootp/bootp.secs'\n",
|
|
" # 'bootp/bootp.type'\n",
|
|
" 'classicstun.length',\n",
|
|
" 'classicstun.type',\n",
|
|
" # 'frame/frame.cap_len'\n",
|
|
" 'frame/frame.number', # Nur für Labeling\n",
|
|
" 'frame/frame.encap_type',\n",
|
|
" 'frame/frame.ignore',\n",
|
|
" 'frame/frame.len',\n",
|
|
" 'frame/frame.marked',\n",
|
|
" 'frame/frame.offset_shift',\n",
|
|
" 'frame/frame.packet_flags',\n",
|
|
" # 'geninfo/caplen'\n",
|
|
" # 'geninfo/len'\n",
|
|
" 'geninfo/timestamp',\n",
|
|
" #Data-Link\n",
|
|
" 'eth/eth.dst',\n",
|
|
" 'eth/eth.dst/eth.dst.ig',\n",
|
|
" 'eth/eth.dst/eth.dst.oui',\n",
|
|
" 'eth/eth.dst/eth.lg',\n",
|
|
" 'eth/eth.src',\n",
|
|
" 'eth/eth.src/eth.ig',\n",
|
|
" 'eth/eth.src/eth.lg',\n",
|
|
" 'eth/eth.src/eth.src.oui',\n",
|
|
" 'eth/eth.type',\n",
|
|
" 'arp/arp.hw.type', # Auffälliges Verhalten des Echo (5 Felder aus Paper zurück)\n",
|
|
" 'arp/arp.proto.type',\n",
|
|
" 'arp/arp.hw.size',\n",
|
|
" 'arp/arp.proto.size',\n",
|
|
" 'arp/arp.opcode',\n",
|
|
" #Network\n",
|
|
" 'icmp/data/data.len',\n",
|
|
" 'icmp/icmp.checksum.status',\n",
|
|
" 'icmp/icmp.code',\n",
|
|
" 'icmp/icmp.ident',\n",
|
|
" 'icmp/icmp.resp_in',\n",
|
|
" 'icmp/icmp.resp_to',\n",
|
|
" 'icmp/icmp.seq',\n",
|
|
" 'icmp/icmp.seq_le',\n",
|
|
" 'icmp/icmp.type',\n",
|
|
" 'icmp/icmp.udp/icmp.udp.dstport',\n",
|
|
" 'icmp/icmp.udp/icmp.udp.length',\n",
|
|
" 'icmp/icmp.udp/icmp.udp.srcport',\n",
|
|
" 'igmp/igmp.checksum.status',\n",
|
|
" 'igmp/igmp.max_resp',\n",
|
|
" 'igmp/igmp.maddr',\n",
|
|
" 'igmp/igmp.type',\n",
|
|
" 'ip/<>/ip.options.routeralert/ip.opt.ra',\n",
|
|
" 'ip/<>/ip.options.routeralert/ip.opt.sec_cl',\n",
|
|
" 'ip/<>/ip.options.routeralert/ip.opt.type',\n",
|
|
" 'ip/ip.checksum.status',\n",
|
|
" 'ip/ip.dsfield',\n",
|
|
" 'ip/ip.dsfield/ip.dsfield.dscp',\n",
|
|
" 'ip/ip.dsfield/ip.dsfield.ecn',\n",
|
|
" 'ip/ip.evil_packet',\n",
|
|
" 'ip/ip.flags',\n",
|
|
" 'ip/ip.flags/ip.flags.df',\n",
|
|
" 'ip/ip.flags/ip.flags.mf',\n",
|
|
" 'ip/ip.flags/ip.flags.rb',\n",
|
|
" 'ip/ip.frag_offset',\n",
|
|
" 'ip/ip.hdr_len',\n",
|
|
" 'ip/ip.id',\n",
|
|
" 'ip/ip.len',\n",
|
|
" 'ip/ip.proto',\n",
|
|
" 'ip/ip.src',\n",
|
|
" 'ip/ip.dst',\n",
|
|
" 'ip/ip.ttl',\n",
|
|
" 'ip/ip.version',\n",
|
|
" # Transport\n",
|
|
" 'tcp/tcp.ack',\n",
|
|
" 'tcp/tcp.analysis/tcp.analysis.bytes_in_flight',\n",
|
|
" 'tcp/tcp.analysis/tcp.analysis.push_bytes_sent',\n",
|
|
" 'tcp/tcp.checksum.status',\n",
|
|
" 'tcp/tcp.completeness',\n",
|
|
" 'tcp/tcp.dstport',\n",
|
|
" 'tcp/tcp.flags/tcp.flags.ack',\n",
|
|
" 'tcp/tcp.flags/tcp.flags.cwr',\n",
|
|
" 'tcp/tcp.flags/tcp.flags.ecn',\n",
|
|
" 'tcp/tcp.flags/tcp.flags.fin',\n",
|
|
" 'tcp/tcp.flags/tcp.flags.ns',\n",
|
|
" 'tcp/tcp.flags/tcp.flags.push',\n",
|
|
" 'tcp/tcp.flags/tcp.flags.res',\n",
|
|
" 'tcp/tcp.flags/tcp.flags.reset',\n",
|
|
" 'tcp/tcp.flags/tcp.flags.syn',\n",
|
|
" 'tcp/tcp.flags/tcp.flags.urg',\n",
|
|
" 'tcp/tcp.hdr_len',\n",
|
|
" 'tcp/tcp.len',\n",
|
|
" 'tcp/tcp.nxtseq',\n",
|
|
" 'tcp/tcp.options/tcp.options.mss/tcp.options.mss_val',\n",
|
|
" 'tcp/tcp.options/tcp.options.nop',\n",
|
|
" # 'tcp/tcp.payload' \n",
|
|
" 'tcp/tcp.seq',\n",
|
|
" 'tcp/tcp.srcport',\n",
|
|
" 'tcp/tcp.stream',\n",
|
|
" 'tcp/tcp.urgent_pointer',\n",
|
|
" 'tcp/tcp.window_size',\n",
|
|
" 'tcp/tcp.window_size_scalefactor',\n",
|
|
" 'tcp/tcp.window_size_value',\n",
|
|
" 'udp/udp.checksum.status',\n",
|
|
" 'udp/udp.dstport',\n",
|
|
" 'udp/udp.length',\n",
|
|
" 'udp/udp.srcport',\n",
|
|
" 'udp/udp.stream',\n",
|
|
"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "84b478867fed089b",
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"### PDML #######################################################\n",
|
|
"PDML_FILE = '../dumps/usage_dump.pdml'\n",
|
|
"\n",
|
|
"\n",
|
|
"################################################################\n",
|
|
"\n",
|
|
"def collect_packet_fields(element: xml.etree.ElementTree.Element, prefix: str, packet_fields_: dict) -> dict:\n",
|
|
" tag_name = element.tag\n",
|
|
" name_attr = element.attrib['name']\n",
|
|
" empty_name_attr = False\n",
|
|
" if not name_attr:\n",
|
|
" empty_name_attr = True\n",
|
|
" col_name_ = f'{prefix}/<>' if prefix else '<>'\n",
|
|
" else:\n",
|
|
" col_name_ = f'{prefix}/{name_attr}' if prefix else f'{name_attr}'\n",
|
|
"\n",
|
|
" # if tag_name != 'proto' and col_name_ not in BLACKLISTED_FIELDS:\n",
|
|
" if tag_name != 'proto':\n",
|
|
" if not empty_name_attr:\n",
|
|
" total_col_names.add(col_name_)\n",
|
|
" try:\n",
|
|
" packet_fields_[col_name_] = element.attrib['show']\n",
|
|
" except KeyError:\n",
|
|
" try:\n",
|
|
" packet_fields_[col_name_] = element.attrib['value']\n",
|
|
" except KeyError:\n",
|
|
" packet_fields_[col_name_] = np.nan\n",
|
|
"\n",
|
|
" # Recursively process child elements (subfields)\n",
|
|
" for child in element:\n",
|
|
" packet_fields_.update(collect_packet_fields(child, col_name_, packet_fields_))\n",
|
|
"\n",
|
|
" return packet_fields_\n",
|
|
"\n",
|
|
"\n",
|
|
"print('Parsing PDML file...')\n",
|
|
"tree = ET.parse(PDML_FILE)\n",
|
|
"root = tree.getroot()\n",
|
|
"\n",
|
|
"total_col_names = set()\n",
|
|
"total_pkt_fields = []\n",
|
|
"\n",
|
|
"print('Collecting packet fields...')\n",
|
|
"# Loop over all packets\n",
|
|
"packets = root.findall('packet')\n",
|
|
"for packet in packets:\n",
|
|
" packet_fields = {}\n",
|
|
" # Loop over all protocols\n",
|
|
" protocols = packet.findall('proto')\n",
|
|
" for protocol in protocols:\n",
|
|
" packet_fields.update(collect_packet_fields(protocol, '', {}))\n",
|
|
" total_pkt_fields.append(packet_fields)\n",
|
|
"\n",
|
|
"total_col_names = sorted(total_col_names)\n",
|
|
"total_pkt_fields = sorted(total_pkt_fields, key=lambda x: int(x['frame/frame.number'])) # Sort by frame.number\n",
|
|
"df_data_full = pd.DataFrame(total_pkt_fields)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b2692056de3e85b7",
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_data_full.shape"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "263145b13cd0b10b",
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# All columns with only NaN rows\n",
|
|
"sorted(df_data_full.columns[df_data_full.isna().all()].tolist())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "25481a29eba04c14",
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# merged_features = set(FEATURES_FROM_PAPER + CUSTOM_CHOSEN_FEATURES)\n",
|
|
"features = set(MINIMAL_FEATURES)\n",
|
|
"f_intersection = features & set(df_data_full.columns)\n",
|
|
"features_not_present_in_pdml = features - f_intersection\n",
|
|
"# features_not_present_in_pdml"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "42c984d759e144b",
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# First add all features which are present in the .pdml\n",
|
|
"df_data_final = df_data_full.loc[:, list(f_intersection)]\n",
|
|
"# Then add all features which have been picked and are from the paper, but are not in the .pdml. Fill them with NaNs.\n",
|
|
"df_data_final[list(features_not_present_in_pdml)] = np.nan\n",
|
|
"# Sort columns\n",
|
|
"# df_data_final.sort_index(axis=1, inplace=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3454b5ca3e48ad92",
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# df_data_final.shape"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "5b5914a527e2a483",
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# df_data_final.columns"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c6371cc759696ec3",
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# All columns with only NaN rows\n",
|
|
"# df_data_final.columns[df_data_final.isna().all()].tolist()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7114a6cf74cafef8",
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# df_data_final.to_csv(\"data_final.csv\", index=False, sep='|')\n",
|
|
"df_data_final.to_pickle(\"../echo-dot-activity.pickel\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"outputs": [],
|
|
"source": [],
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"id": "93deee78d784393a"
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.9"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|