{ "cells": [ { "cell_type": "code", "execution_count": 15, "id": "initial_id", "metadata": { "collapsed": true, "ExecuteTime": { "end_time": "2024-01-07T12:15:05.067240201Z", "start_time": "2024-01-07T12:15:04.968225103Z" } }, "outputs": [], "source": [ "import os\n", "import re\n", "import gc\n", "\n", "from functools import partial\n", "\n", "from concurrent.futures import ProcessPoolExecutor\n", "import queue\n", "\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "from tqdm import tqdm\n", "\n", "import networkx as nx\n", "\n", "# constants\n", "devices_root = os.path.abspath('../../')\n", "time_field = 'geninfo/timestamp'\n", "timedelta_field = \"timedelta\"" ] }, { "cell_type": "code", "execution_count": 2, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loaded merged dataset with 2087024 packets\n" ] }, { "data": { "text/plain": "['arp/arp.hw.size',\n 'arp/arp.hw.type',\n 'arp/arp.opcode',\n 'arp/arp.proto.size',\n 'arp/arp.proto.type',\n 'eth/eth.dst',\n 'eth/eth.dst/eth.dst.ig',\n 'eth/eth.dst/eth.dst.oui',\n 'eth/eth.dst/eth.lg',\n 'eth/eth.src',\n 'eth/eth.src/eth.ig',\n 'eth/eth.src/eth.lg',\n 'eth/eth.src/eth.src.oui',\n 'eth/eth.type',\n 'frame/frame.encap_type',\n 'frame/frame.len',\n 'frame/frame.marked',\n 'frame/frame.number',\n 'frame/frame.offset_shift',\n 'frame/frame.time_delta',\n 'frame/frame.time_relative',\n 'geninfo/timestamp',\n 'icmp/data/data.len',\n 'icmp/icmp.checksum.status',\n 'icmp/icmp.code',\n 'icmp/icmp.ident',\n 'icmp/icmp.resp_in',\n 'icmp/icmp.resp_to',\n 'icmp/icmp.seq',\n 'icmp/icmp.seq_le',\n 'icmp/icmp.type',\n 'igmp/igmp.checksum.status',\n 'igmp/igmp.maddr',\n 'igmp/igmp.max_resp',\n 'igmp/igmp.type',\n 'ip/<>/ip.options.routeralert/ip.opt.ra',\n 'ip/<>/ip.options.routeralert/ip.opt.type',\n 'ip/ip.checksum.status',\n 'ip/ip.dsfield',\n 'ip/ip.dsfield/ip.dsfield.dscp',\n 'ip/ip.dsfield/ip.dsfield.ecn',\n 'ip/ip.dst',\n 'ip/ip.flags',\n 'ip/ip.flags/ip.flags.df',\n 'ip/ip.flags/ip.flags.mf',\n 'ip/ip.flags/ip.flags.rb',\n 'ip/ip.frag_offset',\n 'ip/ip.hdr_len',\n 'ip/ip.id',\n 'ip/ip.len',\n 'ip/ip.proto',\n 'ip/ip.src',\n 'ip/ip.ttl',\n 'ip/ip.version',\n 'name',\n 'tcp/tcp.ack',\n 'tcp/tcp.analysis/tcp.analysis.bytes_in_flight',\n 'tcp/tcp.analysis/tcp.analysis.push_bytes_sent',\n 'tcp/tcp.checksum.status',\n 'tcp/tcp.completeness',\n 'tcp/tcp.dstport',\n 'tcp/tcp.flags/tcp.flags.ack',\n 'tcp/tcp.flags/tcp.flags.cwr',\n 'tcp/tcp.flags/tcp.flags.ecn',\n 'tcp/tcp.flags/tcp.flags.fin',\n 'tcp/tcp.flags/tcp.flags.ns',\n 'tcp/tcp.flags/tcp.flags.push',\n 'tcp/tcp.flags/tcp.flags.res',\n 'tcp/tcp.flags/tcp.flags.reset',\n 'tcp/tcp.flags/tcp.flags.syn',\n 'tcp/tcp.flags/tcp.flags.urg',\n 'tcp/tcp.hdr_len',\n 'tcp/tcp.len',\n 'tcp/tcp.nxtseq',\n 'tcp/tcp.options/tcp.options.mss/tcp.options.mss_val',\n 'tcp/tcp.options/tcp.options.nop',\n 'tcp/tcp.seq',\n 'tcp/tcp.srcport',\n 'tcp/tcp.stream',\n 'tcp/tcp.urgent_pointer',\n 'tcp/tcp.window_size',\n 'tcp/tcp.window_size_scalefactor',\n 'tcp/tcp.window_size_value',\n 'timedelta',\n 'timedelta_prev',\n 'udp/udp.checksum.status',\n 'udp/udp.dstport',\n 'udp/udp.length',\n 'udp/udp.srcport',\n 'udp/udp.stream']" }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# read merged dataset\n", "row_limit = None\n", "merged_dataset_path = os.path.join(devices_root, 'idle_merged.pickle')\n", "merged_dataset_full = pd.read_pickle(merged_dataset_path)\n", "# only load a subset of the dataset, free up some memory\n", "if row_limit is not None:\n", " merged_dataset = merged_dataset_full.iloc[:row_limit]\n", " del merged_dataset_full\n", " gc.collect()\n", "else:\n", " merged_dataset = merged_dataset_full\n", "gc.collect()\n", "print(f\"Loaded merged dataset with {len(merged_dataset)} packets\")\n", "list(sorted(merged_dataset.columns))" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-07T12:01:10.447864422Z", "start_time": "2024-01-07T12:01:08.025881608Z" } }, "id": "bc65976a401780a5" }, { "cell_type": "code", "execution_count": 3, "outputs": [ { "data": { "text/plain": " tcp/tcp.flags/tcp.flags.reset arp/arp.proto.size \\\n0 False NaN \n1 False NaN \n2 False NaN \n3 False NaN \n4 False NaN \n... ... ... \n2087019 False 4.0 \n2087020 False 4.0 \n2087021 False NaN \n2087022 False NaN \n2087023 False NaN \n\n tcp/tcp.options/tcp.options.nop tcp/tcp.checksum.status \\\n0 1.0 2.0 \n1 1.0 2.0 \n2 NaN 2.0 \n3 NaN 2.0 \n4 NaN 2.0 \n... ... ... \n2087019 NaN NaN \n2087020 NaN NaN \n2087021 NaN 2.0 \n2087022 NaN 2.0 \n2087023 NaN 2.0 \n\n udp/udp.checksum.status tcp/tcp.nxtseq \\\n0 NaN 1.0 \n1 NaN 1.0 \n2 NaN 70.0 \n3 NaN 70.0 \n4 NaN 70.0 \n... ... ... \n2087019 NaN NaN \n2087020 NaN NaN \n2087021 NaN 136346.0 \n2087022 NaN 133465.0 \n2087023 NaN 136346.0 \n\n tcp/tcp.options/tcp.options.mss/tcp.options.mss_val tcp/tcp.hdr_len \\\n0 NaN 32.0 \n1 NaN 32.0 \n2 NaN 20.0 \n3 NaN 20.0 \n4 NaN 20.0 \n... ... ... \n2087019 NaN NaN \n2087020 NaN NaN \n2087021 NaN 20.0 \n2087022 NaN 20.0 \n2087023 NaN 20.0 \n\n tcp/tcp.stream ip/ip.dst ... arp/arp.hw.type \\\n0 0.0 192.168.1.201 ... NaN \n1 0.0 54.155.16.139 ... NaN \n2 1.0 76.223.105.139 ... NaN \n3 1.0 192.168.1.178 ... NaN \n4 1.0 76.223.105.139 ... NaN \n... ... ... ... ... \n2087019 NaN NaN ... 1.0 \n2087020 NaN NaN ... 1.0 \n2087021 6.0 13.248.240.249 ... NaN \n2087022 6.0 192.168.1.179 ... NaN \n2087023 6.0 13.248.240.249 ... NaN \n\n eth/eth.src frame/frame.time_relative tcp/tcp.urgent_pointer \\\n0 54:83:3a:70:be:a6 0.000000 False \n1 68:3a:48:4b:53:c5 0.060079 False \n2 1c:d6:bd:b5:d3:bb 0.850154 False \n3 54:83:3a:70:be:a6 0.870282 False \n4 1c:d6:bd:b5:d3:bb 0.929322 False \n... ... ... ... \n2087019 d8:f1:5b:d8:08:0c 86398.038770 False \n2087020 d8:f1:5b:d8:08:0c 86398.038852 False \n2087021 1c:d6:bd:b5:d3:bb 86398.146081 False \n2087022 54:83:3a:70:be:a6 86398.165598 False \n2087023 1c:d6:bd:b5:d3:bb 86398.353964 False \n\n icmp/icmp.seq tcp/tcp.flags/tcp.flags.ecn \\\n0 NaN NaN \n1 NaN NaN \n2 NaN NaN \n3 NaN NaN \n4 NaN NaN \n... ... ... \n2087019 NaN NaN \n2087020 NaN NaN \n2087021 NaN NaN \n2087022 NaN NaN \n2087023 NaN NaN \n\n tcp/tcp.flags/tcp.flags.ns \\\n0 NaN \n1 NaN \n2 NaN \n3 NaN \n4 NaN \n... ... \n2087019 NaN \n2087020 NaN \n2087021 NaN \n2087022 NaN \n2087023 NaN \n\n name \\\n0 /home/alex/projects/it-security-praktikum/devi... \n1 /home/alex/projects/it-security-praktikum/devi... \n2 /home/alex/projects/it-security-praktikum/devi... \n3 /home/alex/projects/it-security-praktikum/devi... \n4 /home/alex/projects/it-security-praktikum/devi... \n... ... \n2087019 /home/alex/projects/it-security-praktikum/devi... \n2087020 /home/alex/projects/it-security-praktikum/devi... \n2087021 /home/alex/projects/it-security-praktikum/devi... \n2087022 /home/alex/projects/it-security-praktikum/devi... \n2087023 /home/alex/projects/it-security-praktikum/devi... \n\n timedelta timedelta_prev \n0 0 days 00:00:00 NaT \n1 0 days 00:00:00.060079 0 days 00:00:00.060079 \n2 0 days 00:00:00.850154 0 days 00:00:00.790075 \n3 0 days 00:00:00.870282 0 days 00:00:00.020128 \n4 0 days 00:00:00.929322 0 days 00:00:00.059040 \n... ... ... \n2087019 18 days 22:40:47.543085 0 days 00:00:00.611045 \n2087020 18 days 22:40:47.543167 0 days 00:00:00.000082 \n2087021 18 days 22:40:47.650396 0 days 00:00:00.107229 \n2087022 18 days 22:40:47.669913 0 days 00:00:00.019517 \n2087023 18 days 22:40:47.858279 0 days 00:00:00.188366 \n\n[2087024 rows x 90 columns]", "text/html": "
| \n | tcp/tcp.flags/tcp.flags.reset | \narp/arp.proto.size | \ntcp/tcp.options/tcp.options.nop | \ntcp/tcp.checksum.status | \nudp/udp.checksum.status | \ntcp/tcp.nxtseq | \ntcp/tcp.options/tcp.options.mss/tcp.options.mss_val | \ntcp/tcp.hdr_len | \ntcp/tcp.stream | \nip/ip.dst | \n... | \narp/arp.hw.type | \neth/eth.src | \nframe/frame.time_relative | \ntcp/tcp.urgent_pointer | \nicmp/icmp.seq | \ntcp/tcp.flags/tcp.flags.ecn | \ntcp/tcp.flags/tcp.flags.ns | \nname | \ntimedelta | \ntimedelta_prev | \n
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \nFalse | \nNaN | \n1.0 | \n2.0 | \nNaN | \n1.0 | \nNaN | \n32.0 | \n0.0 | \n192.168.1.201 | \n... | \nNaN | \n54:83:3a:70:be:a6 | \n0.000000 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 00:00:00 | \nNaT | \n
| 1 | \nFalse | \nNaN | \n1.0 | \n2.0 | \nNaN | \n1.0 | \nNaN | \n32.0 | \n0.0 | \n54.155.16.139 | \n... | \nNaN | \n68:3a:48:4b:53:c5 | \n0.060079 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 00:00:00.060079 | \n0 days 00:00:00.060079 | \n
| 2 | \nFalse | \nNaN | \nNaN | \n2.0 | \nNaN | \n70.0 | \nNaN | \n20.0 | \n1.0 | \n76.223.105.139 | \n... | \nNaN | \n1c:d6:bd:b5:d3:bb | \n0.850154 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 00:00:00.850154 | \n0 days 00:00:00.790075 | \n
| 3 | \nFalse | \nNaN | \nNaN | \n2.0 | \nNaN | \n70.0 | \nNaN | \n20.0 | \n1.0 | \n192.168.1.178 | \n... | \nNaN | \n54:83:3a:70:be:a6 | \n0.870282 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 00:00:00.870282 | \n0 days 00:00:00.020128 | \n
| 4 | \nFalse | \nNaN | \nNaN | \n2.0 | \nNaN | \n70.0 | \nNaN | \n20.0 | \n1.0 | \n76.223.105.139 | \n... | \nNaN | \n1c:d6:bd:b5:d3:bb | \n0.929322 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 00:00:00.929322 | \n0 days 00:00:00.059040 | \n
| ... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n
| 2087019 | \nFalse | \n4.0 | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \n... | \n1.0 | \nd8:f1:5b:d8:08:0c | \n86398.038770 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n18 days 22:40:47.543085 | \n0 days 00:00:00.611045 | \n
| 2087020 | \nFalse | \n4.0 | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \n... | \n1.0 | \nd8:f1:5b:d8:08:0c | \n86398.038852 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n18 days 22:40:47.543167 | \n0 days 00:00:00.000082 | \n
| 2087021 | \nFalse | \nNaN | \nNaN | \n2.0 | \nNaN | \n136346.0 | \nNaN | \n20.0 | \n6.0 | \n13.248.240.249 | \n... | \nNaN | \n1c:d6:bd:b5:d3:bb | \n86398.146081 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n18 days 22:40:47.650396 | \n0 days 00:00:00.107229 | \n
| 2087022 | \nFalse | \nNaN | \nNaN | \n2.0 | \nNaN | \n133465.0 | \nNaN | \n20.0 | \n6.0 | \n192.168.1.179 | \n... | \nNaN | \n54:83:3a:70:be:a6 | \n86398.165598 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n18 days 22:40:47.669913 | \n0 days 00:00:00.019517 | \n
| 2087023 | \nFalse | \nNaN | \nNaN | \n2.0 | \nNaN | \n136346.0 | \nNaN | \n20.0 | \n6.0 | \n13.248.240.249 | \n... | \nNaN | \n1c:d6:bd:b5:d3:bb | \n86398.353964 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n18 days 22:40:47.858279 | \n0 days 00:00:00.188366 | \n
2087024 rows × 90 columns
\n| \n | incoming_packets | \nincoming_bytes | \ndiff_to_last_incoming | \ntotal_incoming_per_second | \nincoming_ports | \noutgoing_packets | \noutgoing_bytes | \ndiff_to_last_outgoing | \ntotal_outgoing_per_second | \noutgoing_ports | \n
|---|
| \n | tcp/tcp.flags/tcp.flags.reset | \narp/arp.proto.size | \ntcp/tcp.options/tcp.options.nop | \ntcp/tcp.checksum.status | \nudp/udp.checksum.status | \ntcp/tcp.nxtseq | \ntcp/tcp.options/tcp.options.mss/tcp.options.mss_val | \ntcp/tcp.hdr_len | \ntcp/tcp.stream | \nip/ip.dst | \n... | \narp/arp.hw.type | \neth/eth.src | \nframe/frame.time_relative | \ntcp/tcp.urgent_pointer | \nicmp/icmp.seq | \ntcp/tcp.flags/tcp.flags.ecn | \ntcp/tcp.flags/tcp.flags.ns | \nname | \ntimedelta | \ntimedelta_prev | \n
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 415 | \nFalse | \nNaN | \nNaN | \n2.0 | \nNaN | \n54.0 | \nNaN | \n20.0 | \n11.0 | \n192.168.1.179 | \n... | \nNaN | \n54:83:3a:70:be:a6 | \n127.215088 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 00:02:07.215088 | \n0 days 00:00:00.035614 | \n
| 433 | \nFalse | \n4.0 | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \n... | \n1.0 | \n54:83:3a:70:be:a6 | \n132.260775 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 00:02:12.260775 | \n0 days 00:00:00.172389 | \n
| 1110 | \nFalse | \nNaN | \nNaN | \n2.0 | \nNaN | \n107.0 | \nNaN | \n20.0 | \n11.0 | \n192.168.1.179 | \n... | \nNaN | \n54:83:3a:70:be:a6 | \n327.213130 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 00:05:27.213130 | \n0 days 00:00:00.035623 | \n
| 1131 | \nFalse | \n4.0 | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \n... | \n1.0 | \n54:83:3a:70:be:a6 | \n332.260899 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 00:05:32.260899 | \n0 days 00:00:00.195519 | \n
| 1746 | \nFalse | \nNaN | \nNaN | \n2.0 | \nNaN | \n160.0 | \nNaN | \n20.0 | \n11.0 | \n192.168.1.179 | \n... | \nNaN | \n54:83:3a:70:be:a6 | \n527.214869 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 00:08:47.214869 | \n0 days 00:00:00.035604 | \n
| ... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n
| 98186 | \nFalse | \nNaN | \nNaN | \nNaN | \n2.0 | \nNaN | \nNaN | \nNaN | \nNaN | \n192.168.1.179 | \n... | \nNaN | \n54:83:3a:70:be:a6 | \n26202.615558 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 07:16:42.615558 | \n0 days 00:00:00.002007 | \n
| 98876 | \nFalse | \nNaN | \nNaN | \n2.0 | \nNaN | \n6944.0 | \nNaN | \n20.0 | \n11.0 | \n192.168.1.179 | \n... | \nNaN | \n54:83:3a:70:be:a6 | \n26397.412012 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 07:19:57.412012 | \n0 days 00:00:00.035625 | \n
| 98888 | \nFalse | \n4.0 | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \n... | \n1.0 | \n54:83:3a:70:be:a6 | \n26402.420902 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 07:20:02.420902 | \n0 days 00:00:00.707142 | \n
| 99606 | \nFalse | \nNaN | \nNaN | \n2.0 | \nNaN | \n6997.0 | \nNaN | \n20.0 | \n11.0 | \n192.168.1.179 | \n... | \nNaN | \n54:83:3a:70:be:a6 | \n26597.423086 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 07:23:17.423086 | \n0 days 00:00:00.035641 | \n
| 99623 | \nFalse | \n4.0 | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \nNaN | \n... | \n1.0 | \n54:83:3a:70:be:a6 | \n26602.500880 | \nFalse | \nNaN | \nNaN | \nNaN | \n/home/alex/projects/it-security-praktikum/devi... | \n0 days 07:23:22.500880 | \n0 days 00:00:01.355524 | \n
425 rows × 90 columns
\n