{ "cells": [ { "cell_type": "code", "id": "initial_id", "metadata": { "collapsed": true, "ExecuteTime": { "end_time": "2024-02-25T17:18:52.775982Z", "start_time": "2024-02-25T17:18:51.404581Z" } }, "source": [ "import os\n", "import sys\n", "import joblib\n", "import pickle\n", "\n", "from functools import partial\n", "\n", "from concurrent.futures import ProcessPoolExecutor\n", "import networkx as nx\n", "\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "from tqdm import tqdm\n", "\n", "from graph_based_intrusion_detection.utils import constants\n", "from graph_based_intrusion_detection.packet_processing.processing_functions import process_packets, \\\n", " multi_process_packets\n", "from graph_based_intrusion_detection.packet_processing.packet_inspection import update_port_list\n", "from graph_based_intrusion_detection.packet_processing.graph_processing import update_layer_3_graph, \\\n", " update_layer_2_graph\n", "\n", "from graph_based_intrusion_detection.utils.state_merging import merge_graph, merge_by_key, merge_states\n", "\n", "from graph_based_intrusion_detection.graph_processing.data_extraction import get_info_for_node, extract_connections, \\\n", " create_dataset_for_node\n", "\n", "# constants\n", "devices_root = os.path.abspath('../../../')\n", "time_field = 'geninfo/timestamp'" ], "outputs": [], "execution_count": 1 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T17:19:00.099525Z", "start_time": "2024-02-25T17:18:52.777510Z" } }, "cell_type": "code", "source": [ "# read merged dataset\n", "row_limit = None\n", "idle_dataset_path = os.path.join(devices_root, 'idle_merged.pickle')\n", "activity_dataset_path = os.path.join(devices_root, 'activity_merged.pickle')\n", "attack_dataset_path = os.path.join(devices_root, 'attack_merged.pickle')\n", "\n", "idle_dataset = pd.read_pickle(idle_dataset_path)\n", "activity_dataset = pd.read_pickle(activity_dataset_path)\n", "attack_dataset = pd.read_pickle(attack_dataset_path)\n", "\n", "merged_dataset_full = pd.concat([idle_dataset, activity_dataset, attack_dataset])\n", "merged_dataset = merged_dataset_full\n", "\n", "print(f\"Loaded merged dataset with {len(merged_dataset)} packets\")\n", "list(sorted(merged_dataset.columns))\n" ], "id": "975524b11bb7a469", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loaded merged dataset with 3194477 packets\n" ] }, { "data": { "text/plain": [ "['Activity',\n", " 'Attack',\n", " 'Farbeändern',\n", " 'Helligkeitssteuern',\n", " 'Party',\n", " 'Themen Auswahl',\n", " 'Timer Ein/Aus',\n", " 'Weiß/gelb',\n", " 'Zeitschalter',\n", " 'arp/arp.hw.size',\n", " 'arp/arp.hw.type',\n", " 'arp/arp.opcode',\n", " 'arp/arp.proto.size',\n", " 'arp/arp.proto.type',\n", " 'eth/eth.dst',\n", " 'eth/eth.dst/eth.dst.ig',\n", " 'eth/eth.dst/eth.dst.oui',\n", " 'eth/eth.dst/eth.lg',\n", " 'eth/eth.src',\n", " 'eth/eth.src/eth.ig',\n", " 'eth/eth.src/eth.lg',\n", " 'eth/eth.src/eth.src.oui',\n", " 'eth/eth.type',\n", " 'frame/frame.encap_type',\n", " 'frame/frame.len',\n", " 'frame/frame.marked',\n", " 'frame/frame.number',\n", " 'frame/frame.offset_shift',\n", " 'frame/frame.time_delta',\n", " 'frame/frame.time_relative',\n", " 'geninfo/timestamp',\n", " 'icmp/data/data.len',\n", " 'icmp/icmp.checksum.status',\n", " 'icmp/icmp.code',\n", " 'icmp/icmp.ident',\n", " 'icmp/icmp.resp_in',\n", " 'icmp/icmp.resp_to',\n", " 'icmp/icmp.seq',\n", " 'icmp/icmp.seq_le',\n", " 'icmp/icmp.type',\n", " 'igmp/igmp.checksum.status',\n", " 'igmp/igmp.maddr',\n", " 'igmp/igmp.max_resp',\n", " 'igmp/igmp.type',\n", " 'illuminance_level',\n", " 'ip/<>/ip.options.routeralert/ip.opt.ra',\n", " 'ip/<>/ip.options.routeralert/ip.opt.type',\n", " 'ip/ip.checksum.status',\n", " 'ip/ip.dsfield',\n", " 'ip/ip.dsfield/ip.dsfield.dscp',\n", " 'ip/ip.dsfield/ip.dsfield.ecn',\n", " 'ip/ip.dst',\n", " 'ip/ip.flags',\n", " 'ip/ip.flags/ip.flags.df',\n", " 'ip/ip.flags/ip.flags.mf',\n", " 'ip/ip.flags/ip.flags.rb',\n", " 'ip/ip.frag_offset',\n", " 'ip/ip.hdr_len',\n", " 'ip/ip.id',\n", " 'ip/ip.len',\n", " 'ip/ip.proto',\n", " 'ip/ip.src',\n", " 'ip/ip.ttl',\n", " 'ip/ip.version',\n", " 'name',\n", " 'presence_absence',\n", " 'tcp/tcp.ack',\n", " 'tcp/tcp.analysis/tcp.analysis.bytes_in_flight',\n", " 'tcp/tcp.analysis/tcp.analysis.push_bytes_sent',\n", " 'tcp/tcp.checksum.status',\n", " 'tcp/tcp.completeness',\n", " 'tcp/tcp.dstport',\n", " 'tcp/tcp.flags/tcp.flags.ack',\n", " 'tcp/tcp.flags/tcp.flags.cwr',\n", " 'tcp/tcp.flags/tcp.flags.ecn',\n", " 'tcp/tcp.flags/tcp.flags.fin',\n", " 'tcp/tcp.flags/tcp.flags.ns',\n", " 'tcp/tcp.flags/tcp.flags.push',\n", " 'tcp/tcp.flags/tcp.flags.res',\n", " 'tcp/tcp.flags/tcp.flags.reset',\n", " 'tcp/tcp.flags/tcp.flags.syn',\n", " 'tcp/tcp.flags/tcp.flags.urg',\n", " 'tcp/tcp.hdr_len',\n", " 'tcp/tcp.len',\n", " 'tcp/tcp.nxtseq',\n", " 'tcp/tcp.options/tcp.options.mss/tcp.options.mss_val',\n", " 'tcp/tcp.options/tcp.options.nop',\n", " 'tcp/tcp.seq',\n", " 'tcp/tcp.srcport',\n", " 'tcp/tcp.stream',\n", " 'tcp/tcp.urgent_pointer',\n", " 'tcp/tcp.window_size',\n", " 'tcp/tcp.window_size_scalefactor',\n", " 'tcp/tcp.window_size_value',\n", " 'timedelta',\n", " 'timedelta_prev',\n", " 'udp/udp.checksum.status',\n", " 'udp/udp.dstport',\n", " 'udp/udp.length',\n", " 'udp/udp.srcport',\n", " 'udp/udp.stream']" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 2 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T17:19:08.859198Z", "start_time": "2024-02-25T17:19:00.100616Z" } }, "cell_type": "code", "source": [ "# take a look at value distributions in the columns\n", "merged_dataset.describe()" ], "id": "989b48a2cdd9efa4", "outputs": [ { "data": { "text/plain": [ " arp/arp.proto.size tcp/tcp.options/tcp.options.nop \\\n", "count 897524.0 411029.0 \n", "mean 4.0 1.0 \n", "min 4.0 1.0 \n", "25% 4.0 1.0 \n", "50% 4.0 1.0 \n", "75% 4.0 1.0 \n", "max 4.0 1.0 \n", "std 0.0 0.0 \n", "\n", " tcp/tcp.checksum.status udp/udp.checksum.status tcp/tcp.nxtseq \\\n", "count 1715900.0 532111.0 1.715900e+06 \n", "mean 2.0 2.0 1.500802e+08 \n", "min 2.0 2.0 0.000000e+00 \n", "25% 2.0 2.0 1.000000e+00 \n", "50% 2.0 2.0 1.000000e+00 \n", "75% 2.0 2.0 4.921175e+04 \n", "max 2.0 2.0 4.294967e+09 \n", "std 0.0 0.0 6.462904e+08 \n", "\n", " tcp/tcp.options/tcp.options.mss/tcp.options.mss_val tcp/tcp.hdr_len \\\n", "count 748231.000000 1.715900e+06 \n", "mean 1459.691718 2.489274e+01 \n", "min 265.000000 2.000000e+01 \n", "25% 1460.000000 2.000000e+01 \n", "50% 1460.000000 2.400000e+01 \n", "75% 1460.000000 2.400000e+01 \n", "max 1460.000000 6.000000e+01 \n", "std 10.950512 5.737436e+00 \n", "\n", " tcp/tcp.stream tcp/tcp.window_size_value ip/ip.hdr_len ... \\\n", "count 1.715900e+06 1.715900e+06 2.184924e+06 ... \n", "mean 5.689277e+04 8.823625e+03 2.000152e+01 ... \n", "min 0.000000e+00 0.000000e+00 2.000000e+01 ... \n", "25% 1.500000e+01 5.010000e+02 2.000000e+01 ... \n", "50% 1.027200e+04 1.024000e+03 2.000000e+01 ... \n", "75% 8.584425e+04 5.357000e+03 2.000000e+01 ... \n", "max 2.789480e+05 6.553500e+04 2.400000e+01 ... \n", "std 8.001493e+04 1.786146e+04 7.799376e-02 ... \n", "\n", " ip/<>/ip.options.routeralert/ip.opt.type \\\n", "count 831.0 \n", "mean 148.0 \n", "min 148.0 \n", "25% 148.0 \n", "50% 148.0 \n", "75% 148.0 \n", "max 148.0 \n", "std 0.0 \n", "\n", " tcp/tcp.analysis/tcp.analysis.push_bytes_sent ip/ip.checksum.status \\\n", "count 337799.000000 2184924.0 \n", "mean 2499.803774 2.0 \n", "min 1.000000 2.0 \n", "25% 38.000000 2.0 \n", "50% 69.000000 2.0 \n", "75% 517.000000 2.0 \n", "max 765073.000000 2.0 \n", "std 24401.842325 0.0 \n", "\n", " arp/arp.hw.type frame/frame.time_relative icmp/icmp.seq \\\n", "count 897524.0 3.167472e+06 1350.000000 \n", "mean 1.0 2.819241e+04 1413.825926 \n", "min 1.0 0.000000e+00 0.000000 \n", "25% 1.0 1.684083e+02 1.000000 \n", "50% 1.0 1.976696e+04 91.000000 \n", "75% 1.0 5.255671e+04 259.750000 \n", "max 1.0 8.640009e+04 10354.000000 \n", "std 0.0 2.870577e+04 3185.183304 \n", "\n", " tcp/tcp.flags/tcp.flags.ecn tcp/tcp.flags/tcp.flags.ns \\\n", "count 46442.0 46442.0 \n", "mean 0.0 0.0 \n", "min 0.0 0.0 \n", "25% 0.0 0.0 \n", "50% 0.0 0.0 \n", "75% 0.0 0.0 \n", "max 0.0 0.0 \n", "std 0.0 0.0 \n", "\n", " timedelta timedelta_prev \n", "count 3194467 3194464 \n", "mean 8 days 04:55:11.007770814 0 days 00:00:00.981176132 \n", "min 0 days 00:00:00 0 days 00:00:00 \n", "25% 0 days 06:42:09.002197 0 days 00:00:00.000046 \n", "50% 9 days 12:24:21.461847 0 days 00:00:00.000473 \n", "75% 14 days 04:40:44.513388 0 days 00:00:00.071609 \n", "max 18 days 22:40:47.858279 7 days 21:15:30.172390 \n", "std 6 days 21:06:14.202244411 0 days 00:08:27.290981739 \n", "\n", "[8 rows x 57 columns]" ], "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
arp/arp.proto.sizetcp/tcp.options/tcp.options.noptcp/tcp.checksum.statusudp/udp.checksum.statustcp/tcp.nxtseqtcp/tcp.options/tcp.options.mss/tcp.options.mss_valtcp/tcp.hdr_lentcp/tcp.streamtcp/tcp.window_size_valueip/ip.hdr_len...ip/<>/ip.options.routeralert/ip.opt.typetcp/tcp.analysis/tcp.analysis.push_bytes_sentip/ip.checksum.statusarp/arp.hw.typeframe/frame.time_relativeicmp/icmp.seqtcp/tcp.flags/tcp.flags.ecntcp/tcp.flags/tcp.flags.nstimedeltatimedelta_prev
count897524.0411029.01715900.0532111.01.715900e+06748231.0000001.715900e+061.715900e+061.715900e+062.184924e+06...831.0337799.0000002184924.0897524.03.167472e+061350.00000046442.046442.031944673194464
mean4.01.02.02.01.500802e+081459.6917182.489274e+015.689277e+048.823625e+032.000152e+01...148.02499.8037742.01.02.819241e+041413.8259260.00.08 days 04:55:11.0077708140 days 00:00:00.981176132
min4.01.02.02.00.000000e+00265.0000002.000000e+010.000000e+000.000000e+002.000000e+01...148.01.0000002.01.00.000000e+000.0000000.00.00 days 00:00:000 days 00:00:00
25%4.01.02.02.01.000000e+001460.0000002.000000e+011.500000e+015.010000e+022.000000e+01...148.038.0000002.01.01.684083e+021.0000000.00.00 days 06:42:09.0021970 days 00:00:00.000046
50%4.01.02.02.01.000000e+001460.0000002.400000e+011.027200e+041.024000e+032.000000e+01...148.069.0000002.01.01.976696e+0491.0000000.00.09 days 12:24:21.4618470 days 00:00:00.000473
75%4.01.02.02.04.921175e+041460.0000002.400000e+018.584425e+045.357000e+032.000000e+01...148.0517.0000002.01.05.255671e+04259.7500000.00.014 days 04:40:44.5133880 days 00:00:00.071609
max4.01.02.02.04.294967e+091460.0000006.000000e+012.789480e+056.553500e+042.400000e+01...148.0765073.0000002.01.08.640009e+0410354.0000000.00.018 days 22:40:47.8582797 days 21:15:30.172390
std0.00.00.00.06.462904e+0810.9505125.737436e+008.001493e+041.786146e+047.799376e-02...0.024401.8423250.00.02.870577e+043185.1833040.00.06 days 21:06:14.2022444110 days 00:08:27.290981739
\n", "

8 rows × 57 columns

\n", "
" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 3 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T17:19:08.880696Z", "start_time": "2024-02-25T17:19:08.861308Z" } }, "cell_type": "code", "source": "merged_dataset.head()", "id": "fc7ec5435335a843", "outputs": [ { "data": { "text/plain": [ " tcp/tcp.flags/tcp.flags.reset arp/arp.proto.size \\\n", "0 False NaN \n", "1 False NaN \n", "2 False NaN \n", "3 False NaN \n", "4 False NaN \n", "\n", " tcp/tcp.options/tcp.options.nop tcp/tcp.checksum.status \\\n", "0 1.0 2.0 \n", "1 1.0 2.0 \n", "2 NaN 2.0 \n", "3 NaN 2.0 \n", "4 NaN 2.0 \n", "\n", " udp/udp.checksum.status tcp/tcp.nxtseq \\\n", "0 NaN 1.0 \n", "1 NaN 1.0 \n", "2 NaN 70.0 \n", "3 NaN 70.0 \n", "4 NaN 70.0 \n", "\n", " tcp/tcp.options/tcp.options.mss/tcp.options.mss_val tcp/tcp.hdr_len \\\n", "0 NaN 32.0 \n", "1 NaN 32.0 \n", "2 NaN 20.0 \n", "3 NaN 20.0 \n", "4 NaN 20.0 \n", "\n", " tcp/tcp.stream ip/ip.dst ... Farbeändern Helligkeitssteuern \\\n", "0 0.0 192.168.1.201 ... NaN NaN \n", "1 0.0 54.155.16.139 ... NaN NaN \n", "2 1.0 76.223.105.139 ... NaN NaN \n", "3 1.0 192.168.1.178 ... NaN NaN \n", "4 1.0 76.223.105.139 ... NaN NaN \n", "\n", " Weiß/gelb Themen Auswahl Party Zeitschalter Timer Ein/Aus \\\n", "0 NaN NaN NaN NaN NaN \n", "1 NaN NaN NaN NaN NaN \n", "2 NaN NaN NaN NaN NaN \n", "3 NaN NaN NaN NaN NaN \n", "4 NaN NaN NaN NaN NaN \n", "\n", " presence_absence illuminance_level Attack \n", "0 NaN NaN NaN \n", "1 NaN NaN NaN \n", "2 NaN NaN NaN \n", "3 NaN NaN NaN \n", "4 NaN NaN NaN \n", "\n", "[5 rows x 101 columns]" ], "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
tcp/tcp.flags/tcp.flags.resetarp/arp.proto.sizetcp/tcp.options/tcp.options.noptcp/tcp.checksum.statusudp/udp.checksum.statustcp/tcp.nxtseqtcp/tcp.options/tcp.options.mss/tcp.options.mss_valtcp/tcp.hdr_lentcp/tcp.streamip/ip.dst...FarbeändernHelligkeitssteuernWeiß/gelbThemen AuswahlPartyZeitschalterTimer Ein/Auspresence_absenceilluminance_levelAttack
0FalseNaN1.02.0NaN1.0NaN32.00.0192.168.1.201...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
1FalseNaN1.02.0NaN1.0NaN32.00.054.155.16.139...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2FalseNaNNaN2.0NaN70.0NaN20.01.076.223.105.139...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
3FalseNaNNaN2.0NaN70.0NaN20.01.0192.168.1.178...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
4FalseNaNNaN2.0NaN70.0NaN20.01.076.223.105.139...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
\n", "

5 rows × 101 columns

\n", "
" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 4 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T17:19:10.617913Z", "start_time": "2024-02-25T17:19:08.881777Z" } }, "cell_type": "code", "source": [ "columns_to_ignore = [\n", " 'Activity',\n", " 'Attack',\n", " 'Farbeändern',\n", " 'Helligkeitssteuern',\n", " 'Party',\n", " 'Themen Auswahl',\n", " 'Timer Ein/Aus',\n", " 'Weiß/gelb',\n", " 'Zeitschalter',\n", " 'name',\n", " 'presence_absence',\n", " \"ip/ip.src\",\n", " \"ip/ip.dst\",\n", " \"eth/eth.src\",\n", " \"eth/eth.dst\",\n", " \"geninfo/timestamp\",\n", " \"timedelta\",\n", " \"timedelta_prev\",\n", "]\n", "\n", "columns_to_predict = [\n", " \"Attack\"\n", "]\n", "\n", "list(merged_dataset.drop(columns=columns_to_ignore, inplace=False).dtypes)" ], "id": "38bcc594f441b716", "outputs": [ { "data": { "text/plain": [ "[dtype('bool'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('O'),\n", " dtype('bool'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('O'),\n", " dtype('float64'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('bool'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('O'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('O'),\n", " dtype('float64'),\n", " dtype('bool'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('O'),\n", " dtype('float64'),\n", " dtype('O'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('O'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('O'),\n", " dtype('O'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('bool'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('float64'),\n", " dtype('O')]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 5 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T17:23:55.928387Z", "start_time": "2024-02-25T17:23:55.817968Z" } }, "cell_type": "code", "source": "len(merged_dataset[merged_dataset[\"Attack\"] == 1])", "id": "42735976a9d79bef", "outputs": [ { "data": { "text/plain": [ "581" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 8 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T17:19:19.505370Z", "start_time": "2024-02-25T17:19:10.619581Z" } }, "cell_type": "code", "source": [ "# prepare dataset for analysis\n", "# drop all unecessary columns\n", "dataset_to_use = merged_dataset.drop(columns=columns_to_ignore + columns_to_predict, inplace=False)\n", "# also drop all object columns\n", "dataset_to_use = dataset_to_use.select_dtypes(exclude=['object'])\n", "\n", "# fill NaN values with 0\n", "dataset_to_use.fillna(0, inplace=True)\n", "\n", "dataset_y = merged_dataset[columns_to_predict[0]].fillna(0).astype(int)" ], "id": "d2dc349a660c2e8a", "outputs": [], "execution_count": 6 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T17:13:29.479253Z", "start_time": "2024-02-25T17:13:29.004170Z" } }, "cell_type": "code", "source": "merged_dataset[columns_to_predict[0]].fillna(0).astype(int)", "id": "fc6bba379e5fe8cd", "outputs": [ { "data": { "text/plain": [ "0 0\n", "1 0\n", "2 0\n", "3 0\n", "4 0\n", " ..\n", "1002184 0\n", "1002185 0\n", "1002186 0\n", "1002187 1\n", "1002188 0\n", "Name: Attack, Length: 3194477, dtype: int64" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 29 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T17:23:17.250619Z", "start_time": "2024-02-25T17:19:25.640575Z" } }, "cell_type": "code", "source": [ "from imblearn.ensemble import BalancedRandomForestClassifier\n", "from sklearn.ensemble import RandomForestClassifier\n", "\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import classification_report, confusion_matrix\n", "\n", "X_train, X_test, y_train, y_test = train_test_split(dataset_to_use,\n", " dataset_y,\n", " test_size=0.33,\n", " random_state=42)\n", "\n", "brf_clf = RandomForestClassifier(n_estimators=100, random_state=42, class_weight='balanced')\n", "brf_clf.fit(X_train, y_train)\n", "\n", "y_pred = brf_clf.predict(X_test)\n", "\n", "print(confusion_matrix(y_test, y_pred))\n" ], "id": "908490c141f16963", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[[1053969 4]\n", " [ 36 169]]\n" ] } ], "execution_count": 7 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T14:32:09.084082Z", "start_time": "2024-02-25T14:32:08.672777Z" } }, "cell_type": "code", "source": [ "def get_ip_nodes(packet: dict | pd.Series, state=None):\n", " if state is None:\n", " state = set()\n", "\n", " if \"ip/ip.src\" in packet:\n", " src = packet[\"ip/ip.src\"]\n", " if src is not None and not pd.isna(src):\n", " state.add(packet[\"ip/ip.src\"])\n", " if \"ip/ip.dst\" in packet:\n", " dst = packet[\"ip/ip.dst\"]\n", " if dst is not None and not pd.isna(dst):\n", " state.add(packet[\"ip/ip.dst\"])\n", "\n", " return state\n", "\n", "\n", "limit = 10_000\n", "state = process_packets(merged_dataset[:limit], [get_ip_nodes], verbose=True)\n", "state" ], "id": "bc220c885b374cb9", "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10000/10000 [00:00<00:00, 531840.13it/s]\n" ] }, { "data": { "text/plain": [ "{'0.0.0.0',\n", " '144.76.59.37',\n", " '168.119.4.163',\n", " '172.104.134.72',\n", " '18.157.203.183',\n", " '18.184.180.174',\n", " '18.195.249.137',\n", " '185.248.189.10',\n", " '192.168.1.1',\n", " '192.168.1.103',\n", " '192.168.1.146',\n", " '192.168.1.178',\n", " '192.168.1.179',\n", " '192.168.1.196',\n", " '192.168.1.201',\n", " '192.168.1.203',\n", " '192.168.1.212',\n", " '192.168.1.218',\n", " '192.168.1.219',\n", " '192.168.1.222',\n", " '192.168.1.234',\n", " '192.168.1.237',\n", " '192.168.1.244',\n", " '192.53.103.108',\n", " '20.56.117.213',\n", " '216.239.35.12',\n", " '224.0.0.1',\n", " '224.0.0.251',\n", " '224.0.1.187',\n", " '255.255.255.255',\n", " '3.86.119.99',\n", " '34.140.193.14',\n", " '34.239.12.200',\n", " '35.158.31.135',\n", " '35.173.146.140',\n", " '52.205.6.197',\n", " '52.215.134.83',\n", " '52.57.38.165',\n", " '52.58.249.45',\n", " '54.155.16.139',\n", " '54.157.113.19',\n", " '76.223.105.139',\n", " '99.84.146.103',\n", " '99.84.146.12',\n", " '99.84.146.13',\n", " '99.84.146.36'}" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 11 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T14:36:08.650369Z", "start_time": "2024-02-25T14:36:08.092347Z" } }, "cell_type": "code", "source": [ "def split_into_connections(packet: dict | pd.Series, state=None):\n", " if \"ip/ip.src\" in packet and \"ip/ip.dst\" in packet:\n", " src = packet[\"ip/ip.src\"]\n", " dst = packet[\"ip/ip.dst\"]\n", " else:\n", " # discard packet\n", " return state\n", "\n", " connection = (src, dst)\n", " if state is None:\n", " state = {}\n", "\n", " if connection not in state:\n", " state[connection] = [packet]\n", " else:\n", " state[connection].append(packet)\n", "\n", " return state\n", "\n", "\n", "processing_functions = [\n", " split_into_connections\n", "]\n", "\n", "limit = 10_000\n", "state = process_packets(merged_dataset.iloc[:limit], processing_functions, verbose=True)" ], "id": "a7df205bfbb31fb0", "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10000/10000 [00:00<00:00, 341909.30it/s]\n" ] } ], "execution_count": 15 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T14:36:40.554999Z", "start_time": "2024-02-25T14:36:40.514684Z" } }, "cell_type": "code", "source": "pd.DataFrame(state[list(state.keys())[0]])", "id": "1e2f7bb4352cd00a", "outputs": [ { "data": { "text/plain": [ " tcp/tcp.flags/tcp.flags.reset arp/arp.proto.size \\\n", "0 False NaN \n", "1 False NaN \n", "2 False NaN \n", "3 False NaN \n", "4 False NaN \n", ".. ... ... \n", "253 False NaN \n", "254 False NaN \n", "255 False NaN \n", "256 False NaN \n", "257 False NaN \n", "\n", " tcp/tcp.options/tcp.options.nop tcp/tcp.checksum.status \\\n", "0 1.0 2.0 \n", "1 1.0 2.0 \n", "2 1.0 2.0 \n", "3 1.0 2.0 \n", "4 1.0 2.0 \n", ".. ... ... \n", "253 1.0 2.0 \n", "254 1.0 2.0 \n", "255 1.0 2.0 \n", "256 1.0 2.0 \n", "257 1.0 2.0 \n", "\n", " udp/udp.checksum.status tcp/tcp.nxtseq \\\n", "0 NaN 1.0 \n", "1 NaN 41.0 \n", "2 NaN 88.0 \n", "3 NaN 87.0 \n", "4 NaN 127.0 \n", ".. ... ... \n", "253 NaN 8987.0 \n", "254 NaN 9027.0 \n", "255 NaN 9026.0 \n", "256 NaN 9066.0 \n", "257 NaN 9065.0 \n", "\n", " tcp/tcp.options/tcp.options.mss/tcp.options.mss_val tcp/tcp.hdr_len \\\n", "0 NaN 32.0 \n", "1 NaN 32.0 \n", "2 NaN 32.0 \n", "3 NaN 32.0 \n", "4 NaN 32.0 \n", ".. ... ... \n", "253 NaN 32.0 \n", "254 NaN 32.0 \n", "255 NaN 32.0 \n", "256 NaN 32.0 \n", "257 NaN 32.0 \n", "\n", " tcp/tcp.stream ip/ip.dst ... Farbeändern Helligkeitssteuern \\\n", "0 0.0 192.168.1.201 ... NaN NaN \n", "1 0.0 192.168.1.201 ... NaN NaN \n", "2 0.0 192.168.1.201 ... NaN NaN \n", "3 0.0 192.168.1.201 ... NaN NaN \n", "4 0.0 192.168.1.201 ... NaN NaN \n", ".. ... ... ... ... ... \n", "253 0.0 192.168.1.201 ... NaN NaN \n", "254 0.0 192.168.1.201 ... NaN NaN \n", "255 0.0 192.168.1.201 ... NaN NaN \n", "256 0.0 192.168.1.201 ... NaN NaN \n", "257 0.0 192.168.1.201 ... NaN NaN \n", "\n", " Weiß/gelb Themen Auswahl Party Zeitschalter Timer Ein/Aus \\\n", "0 NaN NaN NaN NaN NaN \n", "1 NaN NaN NaN NaN NaN \n", "2 NaN NaN NaN NaN NaN \n", "3 NaN NaN NaN NaN NaN \n", "4 NaN NaN NaN NaN NaN \n", ".. ... ... ... ... ... \n", "253 NaN NaN NaN NaN NaN \n", "254 NaN NaN NaN NaN NaN \n", "255 NaN NaN NaN NaN NaN \n", "256 NaN NaN NaN NaN NaN \n", "257 NaN NaN NaN NaN NaN \n", "\n", " presence_absence illuminance_level Attack \n", "0 NaN NaN NaN \n", "1 NaN NaN NaN \n", "2 NaN NaN NaN \n", "3 NaN NaN NaN \n", "4 NaN NaN NaN \n", ".. ... ... ... \n", "253 NaN NaN NaN \n", "254 NaN NaN NaN \n", "255 NaN NaN NaN \n", "256 NaN NaN NaN \n", "257 NaN NaN NaN \n", "\n", "[258 rows x 101 columns]" ], "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
tcp/tcp.flags/tcp.flags.resetarp/arp.proto.sizetcp/tcp.options/tcp.options.noptcp/tcp.checksum.statusudp/udp.checksum.statustcp/tcp.nxtseqtcp/tcp.options/tcp.options.mss/tcp.options.mss_valtcp/tcp.hdr_lentcp/tcp.streamip/ip.dst...FarbeändernHelligkeitssteuernWeiß/gelbThemen AuswahlPartyZeitschalterTimer Ein/Auspresence_absenceilluminance_levelAttack
0FalseNaN1.02.0NaN1.0NaN32.00.0192.168.1.201...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
1FalseNaN1.02.0NaN41.0NaN32.00.0192.168.1.201...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2FalseNaN1.02.0NaN88.0NaN32.00.0192.168.1.201...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
3FalseNaN1.02.0NaN87.0NaN32.00.0192.168.1.201...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
4FalseNaN1.02.0NaN127.0NaN32.00.0192.168.1.201...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
..................................................................
253FalseNaN1.02.0NaN8987.0NaN32.00.0192.168.1.201...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
254FalseNaN1.02.0NaN9027.0NaN32.00.0192.168.1.201...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
255FalseNaN1.02.0NaN9026.0NaN32.00.0192.168.1.201...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
256FalseNaN1.02.0NaN9066.0NaN32.00.0192.168.1.201...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
257FalseNaN1.02.0NaN9065.0NaN32.00.0192.168.1.201...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
\n", "

258 rows × 101 columns

\n", "
" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 18 } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.6" } }, "nbformat": 4, "nbformat_minor": 5 }