{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55bc224f879f844a",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import xml.etree.ElementTree\n",
    "import xml.etree.ElementTree as ET\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "DEVICES_LAB = {\n",
    "    # 'a8:03:2a:b1:35:60': 'dev01-shelly-plus-1pm-relais', # Christoph\n",
    "    # '60:01:94:c7:69:ac': 'dev02-sonoff-relais', # Oliver\n",
    "    # 'd8:f1:5b:d8:08:0c': 'dev03-tuya-lampe', # Leonard\n",
    "    # '1c:d6:bd:b5:d3:bb': 'dev04-linkind-zigbee-mini-hub', # Thorsten\n",
    "    # '6c:5a:b0:7d:e2:25': 'dev05-tplink-tapo-l530e-birne', # Suad\n",
    "    # '54:af:97:7c:5e:f0': 'dev06-tapo-steckdose', # Andreas\n",
    "    # '74:ab:93:de:a0:7e': 'dev07-blink-sicherheitskamera', # Julian\n",
    "    '34:25:be:ef:91:bf': 'dev08-echo-dot-l4s3re',  # Alexander\n",
    "    # '70:ee:50:90:64:04': 'dev09-netatmo-smart-weather-station', # Jakobus\n",
    "    # '68:3a:48:4b:53:c5': 'dev10-aeotec-z-wave-hub', # Andre\n",
    "    # 'dc:ed:83:4a:cf:76': 'dev11-aqara-presence-sensor-fp2', # Artur\n",
    "    # '8c:f6:81:dc:63:54': 'dev12-shelly-bewegungsmelder', # Moritz\n",
    "    # '24:4c:ab:43:0d:0f': 'dev13-shelly-flood',  # Mohamad\n",
    "    # '08:b6:1f:cc:4d:c0': 'dev14-shelly-ht-temperatur-sensor', # Bastian\n",
    "    # '90:48:6c:17:ae:25': 'dev15-ring-door-camera' # Victor (Ring hinzugefügt, device Nummern angepasst)\n",
    "}"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "b5715a859aec218c"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39ad609a78e03fa0",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "MINIMAL_FEATURES = [\n",
    "    #Meta\n",
    "    '_ws.malformed/_ws.expert/_ws.malformed.expert',\n",
    "    # 'bootp/bootp.dhcp'\n",
    "    # 'bootp/bootp.flags.bc'\n",
    "    # 'bootp/bootp.flags.reserved'\n",
    "    # 'bootp/bootp.hops'\n",
    "    # 'bootp/bootp.hw.len'\n",
    "    # 'bootp/bootp.hw.type'\n",
    "    # 'bootp/bootp.secs'\n",
    "    # 'bootp/bootp.type'\n",
    "    'classicstun.length',\n",
    "    'classicstun.type',\n",
    "    # 'frame/frame.cap_len'\n",
    "    'frame/frame.number',  # Nur für Labeling\n",
    "    'frame/frame.encap_type',\n",
    "    'frame/frame.ignore',\n",
    "    'frame/frame.len',\n",
    "    'frame/frame.marked',\n",
    "    'frame/frame.offset_shift',\n",
    "    'frame/frame.packet_flags',\n",
    "    # 'geninfo/caplen'\n",
    "    # 'geninfo/len'\n",
    "    'geninfo/timestamp',\n",
    "    #Data-Link\n",
    "    'eth/eth.dst',\n",
    "    'eth/eth.dst/eth.dst.ig',\n",
    "    'eth/eth.dst/eth.dst.oui',\n",
    "    'eth/eth.dst/eth.lg',\n",
    "    'eth/eth.src',\n",
    "    'eth/eth.src/eth.ig',\n",
    "    'eth/eth.src/eth.lg',\n",
    "    'eth/eth.src/eth.src.oui',\n",
    "    'eth/eth.type',\n",
    "    'arp/arp.hw.type',  # Auffälliges Verhalten des Echo (5 Felder aus Paper zurück)\n",
    "    'arp/arp.proto.type',\n",
    "    'arp/arp.hw.size',\n",
    "    'arp/arp.proto.size',\n",
    "    'arp/arp.opcode',\n",
    "    #Network\n",
    "    'icmp/data/data.len',\n",
    "    'icmp/icmp.checksum.status',\n",
    "    'icmp/icmp.code',\n",
    "    'icmp/icmp.ident',\n",
    "    'icmp/icmp.resp_in',\n",
    "    'icmp/icmp.resp_to',\n",
    "    'icmp/icmp.seq',\n",
    "    'icmp/icmp.seq_le',\n",
    "    'icmp/icmp.type',\n",
    "    'icmp/icmp.udp/icmp.udp.dstport',\n",
    "    'icmp/icmp.udp/icmp.udp.length',\n",
    "    'icmp/icmp.udp/icmp.udp.srcport',\n",
    "    'igmp/igmp.checksum.status',\n",
    "    'igmp/igmp.max_resp',\n",
    "    'igmp/igmp.maddr',\n",
    "    'igmp/igmp.type',\n",
    "    'ip/<>/ip.options.routeralert/ip.opt.ra',\n",
    "    'ip/<>/ip.options.routeralert/ip.opt.sec_cl',\n",
    "    'ip/<>/ip.options.routeralert/ip.opt.type',\n",
    "    'ip/ip.checksum.status',\n",
    "    'ip/ip.dsfield',\n",
    "    'ip/ip.dsfield/ip.dsfield.dscp',\n",
    "    'ip/ip.dsfield/ip.dsfield.ecn',\n",
    "    'ip/ip.evil_packet',\n",
    "    'ip/ip.flags',\n",
    "    'ip/ip.flags/ip.flags.df',\n",
    "    'ip/ip.flags/ip.flags.mf',\n",
    "    'ip/ip.flags/ip.flags.rb',\n",
    "    'ip/ip.frag_offset',\n",
    "    'ip/ip.hdr_len',\n",
    "    'ip/ip.id',\n",
    "    'ip/ip.len',\n",
    "    'ip/ip.proto',\n",
    "    'ip/ip.src',\n",
    "    'ip/ip.dst',\n",
    "    'ip/ip.ttl',\n",
    "    'ip/ip.version',\n",
    "    # Transport\n",
    "    'tcp/tcp.ack',\n",
    "    'tcp/tcp.analysis/tcp.analysis.bytes_in_flight',\n",
    "    'tcp/tcp.analysis/tcp.analysis.push_bytes_sent',\n",
    "    'tcp/tcp.checksum.status',\n",
    "    'tcp/tcp.completeness',\n",
    "    'tcp/tcp.dstport',\n",
    "    'tcp/tcp.flags/tcp.flags.ack',\n",
    "    'tcp/tcp.flags/tcp.flags.cwr',\n",
    "    'tcp/tcp.flags/tcp.flags.ecn',\n",
    "    'tcp/tcp.flags/tcp.flags.fin',\n",
    "    'tcp/tcp.flags/tcp.flags.ns',\n",
    "    'tcp/tcp.flags/tcp.flags.push',\n",
    "    'tcp/tcp.flags/tcp.flags.res',\n",
    "    'tcp/tcp.flags/tcp.flags.reset',\n",
    "    'tcp/tcp.flags/tcp.flags.syn',\n",
    "    'tcp/tcp.flags/tcp.flags.urg',\n",
    "    'tcp/tcp.hdr_len',\n",
    "    'tcp/tcp.len',\n",
    "    'tcp/tcp.nxtseq',\n",
    "    'tcp/tcp.options/tcp.options.mss/tcp.options.mss_val',\n",
    "    'tcp/tcp.options/tcp.options.nop',\n",
    "    # 'tcp/tcp.payload' \n",
    "    'tcp/tcp.seq',\n",
    "    'tcp/tcp.srcport',\n",
    "    'tcp/tcp.stream',\n",
    "    'tcp/tcp.urgent_pointer',\n",
    "    'tcp/tcp.window_size',\n",
    "    'tcp/tcp.window_size_scalefactor',\n",
    "    'tcp/tcp.window_size_value',\n",
    "    'udp/udp.checksum.status',\n",
    "    'udp/udp.dstport',\n",
    "    'udp/udp.length',\n",
    "    'udp/udp.srcport',\n",
    "    'udp/udp.stream',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84b478867fed089b",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "### PDML #######################################################\n",
    "PDML_FILE = '../dumps/usage_dump.pdml'\n",
    "\n",
    "\n",
    "################################################################\n",
    "\n",
    "def collect_packet_fields(element: xml.etree.ElementTree.Element, prefix: str, packet_fields_: dict) -> dict:\n",
    "    tag_name = element.tag\n",
    "    name_attr = element.attrib['name']\n",
    "    empty_name_attr = False\n",
    "    if not name_attr:\n",
    "        empty_name_attr = True\n",
    "        col_name_ = f'{prefix}/<>' if prefix else '<>'\n",
    "    else:\n",
    "        col_name_ = f'{prefix}/{name_attr}' if prefix else f'{name_attr}'\n",
    "\n",
    "    # if tag_name != 'proto' and col_name_ not in BLACKLISTED_FIELDS:\n",
    "    if tag_name != 'proto':\n",
    "        if not empty_name_attr:\n",
    "            total_col_names.add(col_name_)\n",
    "            try:\n",
    "                packet_fields_[col_name_] = element.attrib['show']\n",
    "            except KeyError:\n",
    "                try:\n",
    "                    packet_fields_[col_name_] = element.attrib['value']\n",
    "                except KeyError:\n",
    "                    packet_fields_[col_name_] = np.nan\n",
    "\n",
    "    # Recursively process child elements (subfields)\n",
    "    for child in element:\n",
    "        packet_fields_.update(collect_packet_fields(child, col_name_, packet_fields_))\n",
    "\n",
    "    return packet_fields_\n",
    "\n",
    "\n",
    "print('Parsing PDML file...')\n",
    "tree = ET.parse(PDML_FILE)\n",
    "root = tree.getroot()\n",
    "\n",
    "total_col_names = set()\n",
    "total_pkt_fields = []\n",
    "\n",
    "print('Collecting packet fields...')\n",
    "# Loop over all packets\n",
    "packets = root.findall('packet')\n",
    "for packet in packets:\n",
    "    packet_fields = {}\n",
    "    # Loop over all protocols\n",
    "    protocols = packet.findall('proto')\n",
    "    for protocol in protocols:\n",
    "        packet_fields.update(collect_packet_fields(protocol, '', {}))\n",
    "    total_pkt_fields.append(packet_fields)\n",
    "\n",
    "total_col_names = sorted(total_col_names)\n",
    "total_pkt_fields = sorted(total_pkt_fields, key=lambda x: int(x['frame/frame.number']))  # Sort by frame.number\n",
    "df_data_full = pd.DataFrame(total_pkt_fields)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2692056de3e85b7",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_data_full.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "263145b13cd0b10b",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# All columns with only NaN rows\n",
    "sorted(df_data_full.columns[df_data_full.isna().all()].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25481a29eba04c14",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# merged_features = set(FEATURES_FROM_PAPER + CUSTOM_CHOSEN_FEATURES)\n",
    "features = set(MINIMAL_FEATURES)\n",
    "f_intersection = features & set(df_data_full.columns)\n",
    "features_not_present_in_pdml = features - f_intersection\n",
    "# features_not_present_in_pdml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42c984d759e144b",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# First add all features which are present in the .pdml\n",
    "df_data_final = df_data_full.loc[:, list(f_intersection)]\n",
    "# Then add all features which have been picked and are from the paper, but are not in the .pdml. Fill them with NaNs.\n",
    "df_data_final[list(features_not_present_in_pdml)] = np.nan\n",
    "# Sort columns\n",
    "# df_data_final.sort_index(axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3454b5ca3e48ad92",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# df_data_final.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b5914a527e2a483",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# df_data_final.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6371cc759696ec3",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# All columns with only NaN rows\n",
    "# df_data_final.columns[df_data_final.isna().all()].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7114a6cf74cafef8",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# df_data_final.to_csv(\"data_final.csv\", index=False, sep='|')\n",
    "df_data_final.to_pickle(\"../echo-dot-activity.pickel\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   },
   "id": "93deee78d784393a"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}