1643 lines
58 KiB
Plaintext
1643 lines
58 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"id": "initial_id",
|
||
"metadata": {
|
||
"collapsed": true,
|
||
"ExecuteTime": {
|
||
"end_time": "2024-02-25T17:18:52.775982Z",
|
||
"start_time": "2024-02-25T17:18:51.404581Z"
|
||
}
|
||
},
|
||
"source": [
|
||
"import os\n",
|
||
"import sys\n",
|
||
"import joblib\n",
|
||
"import pickle\n",
|
||
"\n",
|
||
"from functools import partial\n",
|
||
"\n",
|
||
"from concurrent.futures import ProcessPoolExecutor\n",
|
||
"import networkx as nx\n",
|
||
"\n",
|
||
"import numpy as np\n",
|
||
"import pandas as pd\n",
|
||
"import matplotlib.pyplot as plt\n",
|
||
"from tqdm import tqdm\n",
|
||
"\n",
|
||
"from graph_based_intrusion_detection.utils import constants\n",
|
||
"from graph_based_intrusion_detection.packet_processing.processing_functions import process_packets, \\\n",
|
||
" multi_process_packets\n",
|
||
"from graph_based_intrusion_detection.packet_processing.packet_inspection import update_port_list\n",
|
||
"from graph_based_intrusion_detection.packet_processing.graph_processing import update_layer_3_graph, \\\n",
|
||
" update_layer_2_graph\n",
|
||
"\n",
|
||
"from graph_based_intrusion_detection.utils.state_merging import merge_graph, merge_by_key, merge_states\n",
|
||
"\n",
|
||
"from graph_based_intrusion_detection.graph_processing.data_extraction import get_info_for_node, extract_connections, \\\n",
|
||
" create_dataset_for_node\n",
|
||
"\n",
|
||
"# constants\n",
|
||
"devices_root = os.path.abspath('../../../')\n",
|
||
"time_field = 'geninfo/timestamp'"
|
||
],
|
||
"outputs": [],
|
||
"execution_count": 1
|
||
},
|
||
{
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2024-02-25T17:19:00.099525Z",
|
||
"start_time": "2024-02-25T17:18:52.777510Z"
|
||
}
|
||
},
|
||
"cell_type": "code",
|
||
"source": [
|
||
"# read merged dataset\n",
|
||
"row_limit = None\n",
|
||
"idle_dataset_path = os.path.join(devices_root, 'idle_merged.pickle')\n",
|
||
"activity_dataset_path = os.path.join(devices_root, 'activity_merged.pickle')\n",
|
||
"attack_dataset_path = os.path.join(devices_root, 'attack_merged.pickle')\n",
|
||
"\n",
|
||
"idle_dataset = pd.read_pickle(idle_dataset_path)\n",
|
||
"activity_dataset = pd.read_pickle(activity_dataset_path)\n",
|
||
"attack_dataset = pd.read_pickle(attack_dataset_path)\n",
|
||
"\n",
|
||
"merged_dataset_full = pd.concat([idle_dataset, activity_dataset, attack_dataset])\n",
|
||
"merged_dataset = merged_dataset_full\n",
|
||
"\n",
|
||
"print(f\"Loaded merged dataset with {len(merged_dataset)} packets\")\n",
|
||
"list(sorted(merged_dataset.columns))\n"
|
||
],
|
||
"id": "975524b11bb7a469",
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Loaded merged dataset with 3194477 packets\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"['Activity',\n",
|
||
" 'Attack',\n",
|
||
" 'Farbeändern',\n",
|
||
" 'Helligkeitssteuern',\n",
|
||
" 'Party',\n",
|
||
" 'Themen Auswahl',\n",
|
||
" 'Timer Ein/Aus',\n",
|
||
" 'Weiß/gelb',\n",
|
||
" 'Zeitschalter',\n",
|
||
" 'arp/arp.hw.size',\n",
|
||
" 'arp/arp.hw.type',\n",
|
||
" 'arp/arp.opcode',\n",
|
||
" 'arp/arp.proto.size',\n",
|
||
" 'arp/arp.proto.type',\n",
|
||
" 'eth/eth.dst',\n",
|
||
" 'eth/eth.dst/eth.dst.ig',\n",
|
||
" 'eth/eth.dst/eth.dst.oui',\n",
|
||
" 'eth/eth.dst/eth.lg',\n",
|
||
" 'eth/eth.src',\n",
|
||
" 'eth/eth.src/eth.ig',\n",
|
||
" 'eth/eth.src/eth.lg',\n",
|
||
" 'eth/eth.src/eth.src.oui',\n",
|
||
" 'eth/eth.type',\n",
|
||
" 'frame/frame.encap_type',\n",
|
||
" 'frame/frame.len',\n",
|
||
" 'frame/frame.marked',\n",
|
||
" 'frame/frame.number',\n",
|
||
" 'frame/frame.offset_shift',\n",
|
||
" 'frame/frame.time_delta',\n",
|
||
" 'frame/frame.time_relative',\n",
|
||
" 'geninfo/timestamp',\n",
|
||
" 'icmp/data/data.len',\n",
|
||
" 'icmp/icmp.checksum.status',\n",
|
||
" 'icmp/icmp.code',\n",
|
||
" 'icmp/icmp.ident',\n",
|
||
" 'icmp/icmp.resp_in',\n",
|
||
" 'icmp/icmp.resp_to',\n",
|
||
" 'icmp/icmp.seq',\n",
|
||
" 'icmp/icmp.seq_le',\n",
|
||
" 'icmp/icmp.type',\n",
|
||
" 'igmp/igmp.checksum.status',\n",
|
||
" 'igmp/igmp.maddr',\n",
|
||
" 'igmp/igmp.max_resp',\n",
|
||
" 'igmp/igmp.type',\n",
|
||
" 'illuminance_level',\n",
|
||
" 'ip/<>/ip.options.routeralert/ip.opt.ra',\n",
|
||
" 'ip/<>/ip.options.routeralert/ip.opt.type',\n",
|
||
" 'ip/ip.checksum.status',\n",
|
||
" 'ip/ip.dsfield',\n",
|
||
" 'ip/ip.dsfield/ip.dsfield.dscp',\n",
|
||
" 'ip/ip.dsfield/ip.dsfield.ecn',\n",
|
||
" 'ip/ip.dst',\n",
|
||
" 'ip/ip.flags',\n",
|
||
" 'ip/ip.flags/ip.flags.df',\n",
|
||
" 'ip/ip.flags/ip.flags.mf',\n",
|
||
" 'ip/ip.flags/ip.flags.rb',\n",
|
||
" 'ip/ip.frag_offset',\n",
|
||
" 'ip/ip.hdr_len',\n",
|
||
" 'ip/ip.id',\n",
|
||
" 'ip/ip.len',\n",
|
||
" 'ip/ip.proto',\n",
|
||
" 'ip/ip.src',\n",
|
||
" 'ip/ip.ttl',\n",
|
||
" 'ip/ip.version',\n",
|
||
" 'name',\n",
|
||
" 'presence_absence',\n",
|
||
" 'tcp/tcp.ack',\n",
|
||
" 'tcp/tcp.analysis/tcp.analysis.bytes_in_flight',\n",
|
||
" 'tcp/tcp.analysis/tcp.analysis.push_bytes_sent',\n",
|
||
" 'tcp/tcp.checksum.status',\n",
|
||
" 'tcp/tcp.completeness',\n",
|
||
" 'tcp/tcp.dstport',\n",
|
||
" 'tcp/tcp.flags/tcp.flags.ack',\n",
|
||
" 'tcp/tcp.flags/tcp.flags.cwr',\n",
|
||
" 'tcp/tcp.flags/tcp.flags.ecn',\n",
|
||
" 'tcp/tcp.flags/tcp.flags.fin',\n",
|
||
" 'tcp/tcp.flags/tcp.flags.ns',\n",
|
||
" 'tcp/tcp.flags/tcp.flags.push',\n",
|
||
" 'tcp/tcp.flags/tcp.flags.res',\n",
|
||
" 'tcp/tcp.flags/tcp.flags.reset',\n",
|
||
" 'tcp/tcp.flags/tcp.flags.syn',\n",
|
||
" 'tcp/tcp.flags/tcp.flags.urg',\n",
|
||
" 'tcp/tcp.hdr_len',\n",
|
||
" 'tcp/tcp.len',\n",
|
||
" 'tcp/tcp.nxtseq',\n",
|
||
" 'tcp/tcp.options/tcp.options.mss/tcp.options.mss_val',\n",
|
||
" 'tcp/tcp.options/tcp.options.nop',\n",
|
||
" 'tcp/tcp.seq',\n",
|
||
" 'tcp/tcp.srcport',\n",
|
||
" 'tcp/tcp.stream',\n",
|
||
" 'tcp/tcp.urgent_pointer',\n",
|
||
" 'tcp/tcp.window_size',\n",
|
||
" 'tcp/tcp.window_size_scalefactor',\n",
|
||
" 'tcp/tcp.window_size_value',\n",
|
||
" 'timedelta',\n",
|
||
" 'timedelta_prev',\n",
|
||
" 'udp/udp.checksum.status',\n",
|
||
" 'udp/udp.dstport',\n",
|
||
" 'udp/udp.length',\n",
|
||
" 'udp/udp.srcport',\n",
|
||
" 'udp/udp.stream']"
|
||
]
|
||
},
|
||
"execution_count": 2,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"execution_count": 2
|
||
},
|
||
{
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2024-02-25T17:19:08.859198Z",
|
||
"start_time": "2024-02-25T17:19:00.100616Z"
|
||
}
|
||
},
|
||
"cell_type": "code",
|
||
"source": [
|
||
"# take a look at value distributions in the columns\n",
|
||
"merged_dataset.describe()"
|
||
],
|
||
"id": "989b48a2cdd9efa4",
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
" arp/arp.proto.size tcp/tcp.options/tcp.options.nop \\\n",
|
||
"count 897524.0 411029.0 \n",
|
||
"mean 4.0 1.0 \n",
|
||
"min 4.0 1.0 \n",
|
||
"25% 4.0 1.0 \n",
|
||
"50% 4.0 1.0 \n",
|
||
"75% 4.0 1.0 \n",
|
||
"max 4.0 1.0 \n",
|
||
"std 0.0 0.0 \n",
|
||
"\n",
|
||
" tcp/tcp.checksum.status udp/udp.checksum.status tcp/tcp.nxtseq \\\n",
|
||
"count 1715900.0 532111.0 1.715900e+06 \n",
|
||
"mean 2.0 2.0 1.500802e+08 \n",
|
||
"min 2.0 2.0 0.000000e+00 \n",
|
||
"25% 2.0 2.0 1.000000e+00 \n",
|
||
"50% 2.0 2.0 1.000000e+00 \n",
|
||
"75% 2.0 2.0 4.921175e+04 \n",
|
||
"max 2.0 2.0 4.294967e+09 \n",
|
||
"std 0.0 0.0 6.462904e+08 \n",
|
||
"\n",
|
||
" tcp/tcp.options/tcp.options.mss/tcp.options.mss_val tcp/tcp.hdr_len \\\n",
|
||
"count 748231.000000 1.715900e+06 \n",
|
||
"mean 1459.691718 2.489274e+01 \n",
|
||
"min 265.000000 2.000000e+01 \n",
|
||
"25% 1460.000000 2.000000e+01 \n",
|
||
"50% 1460.000000 2.400000e+01 \n",
|
||
"75% 1460.000000 2.400000e+01 \n",
|
||
"max 1460.000000 6.000000e+01 \n",
|
||
"std 10.950512 5.737436e+00 \n",
|
||
"\n",
|
||
" tcp/tcp.stream tcp/tcp.window_size_value ip/ip.hdr_len ... \\\n",
|
||
"count 1.715900e+06 1.715900e+06 2.184924e+06 ... \n",
|
||
"mean 5.689277e+04 8.823625e+03 2.000152e+01 ... \n",
|
||
"min 0.000000e+00 0.000000e+00 2.000000e+01 ... \n",
|
||
"25% 1.500000e+01 5.010000e+02 2.000000e+01 ... \n",
|
||
"50% 1.027200e+04 1.024000e+03 2.000000e+01 ... \n",
|
||
"75% 8.584425e+04 5.357000e+03 2.000000e+01 ... \n",
|
||
"max 2.789480e+05 6.553500e+04 2.400000e+01 ... \n",
|
||
"std 8.001493e+04 1.786146e+04 7.799376e-02 ... \n",
|
||
"\n",
|
||
" ip/<>/ip.options.routeralert/ip.opt.type \\\n",
|
||
"count 831.0 \n",
|
||
"mean 148.0 \n",
|
||
"min 148.0 \n",
|
||
"25% 148.0 \n",
|
||
"50% 148.0 \n",
|
||
"75% 148.0 \n",
|
||
"max 148.0 \n",
|
||
"std 0.0 \n",
|
||
"\n",
|
||
" tcp/tcp.analysis/tcp.analysis.push_bytes_sent ip/ip.checksum.status \\\n",
|
||
"count 337799.000000 2184924.0 \n",
|
||
"mean 2499.803774 2.0 \n",
|
||
"min 1.000000 2.0 \n",
|
||
"25% 38.000000 2.0 \n",
|
||
"50% 69.000000 2.0 \n",
|
||
"75% 517.000000 2.0 \n",
|
||
"max 765073.000000 2.0 \n",
|
||
"std 24401.842325 0.0 \n",
|
||
"\n",
|
||
" arp/arp.hw.type frame/frame.time_relative icmp/icmp.seq \\\n",
|
||
"count 897524.0 3.167472e+06 1350.000000 \n",
|
||
"mean 1.0 2.819241e+04 1413.825926 \n",
|
||
"min 1.0 0.000000e+00 0.000000 \n",
|
||
"25% 1.0 1.684083e+02 1.000000 \n",
|
||
"50% 1.0 1.976696e+04 91.000000 \n",
|
||
"75% 1.0 5.255671e+04 259.750000 \n",
|
||
"max 1.0 8.640009e+04 10354.000000 \n",
|
||
"std 0.0 2.870577e+04 3185.183304 \n",
|
||
"\n",
|
||
" tcp/tcp.flags/tcp.flags.ecn tcp/tcp.flags/tcp.flags.ns \\\n",
|
||
"count 46442.0 46442.0 \n",
|
||
"mean 0.0 0.0 \n",
|
||
"min 0.0 0.0 \n",
|
||
"25% 0.0 0.0 \n",
|
||
"50% 0.0 0.0 \n",
|
||
"75% 0.0 0.0 \n",
|
||
"max 0.0 0.0 \n",
|
||
"std 0.0 0.0 \n",
|
||
"\n",
|
||
" timedelta timedelta_prev \n",
|
||
"count 3194467 3194464 \n",
|
||
"mean 8 days 04:55:11.007770814 0 days 00:00:00.981176132 \n",
|
||
"min 0 days 00:00:00 0 days 00:00:00 \n",
|
||
"25% 0 days 06:42:09.002197 0 days 00:00:00.000046 \n",
|
||
"50% 9 days 12:24:21.461847 0 days 00:00:00.000473 \n",
|
||
"75% 14 days 04:40:44.513388 0 days 00:00:00.071609 \n",
|
||
"max 18 days 22:40:47.858279 7 days 21:15:30.172390 \n",
|
||
"std 6 days 21:06:14.202244411 0 days 00:08:27.290981739 \n",
|
||
"\n",
|
||
"[8 rows x 57 columns]"
|
||
],
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>arp/arp.proto.size</th>\n",
|
||
" <th>tcp/tcp.options/tcp.options.nop</th>\n",
|
||
" <th>tcp/tcp.checksum.status</th>\n",
|
||
" <th>udp/udp.checksum.status</th>\n",
|
||
" <th>tcp/tcp.nxtseq</th>\n",
|
||
" <th>tcp/tcp.options/tcp.options.mss/tcp.options.mss_val</th>\n",
|
||
" <th>tcp/tcp.hdr_len</th>\n",
|
||
" <th>tcp/tcp.stream</th>\n",
|
||
" <th>tcp/tcp.window_size_value</th>\n",
|
||
" <th>ip/ip.hdr_len</th>\n",
|
||
" <th>...</th>\n",
|
||
" <th>ip/<>/ip.options.routeralert/ip.opt.type</th>\n",
|
||
" <th>tcp/tcp.analysis/tcp.analysis.push_bytes_sent</th>\n",
|
||
" <th>ip/ip.checksum.status</th>\n",
|
||
" <th>arp/arp.hw.type</th>\n",
|
||
" <th>frame/frame.time_relative</th>\n",
|
||
" <th>icmp/icmp.seq</th>\n",
|
||
" <th>tcp/tcp.flags/tcp.flags.ecn</th>\n",
|
||
" <th>tcp/tcp.flags/tcp.flags.ns</th>\n",
|
||
" <th>timedelta</th>\n",
|
||
" <th>timedelta_prev</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>count</th>\n",
|
||
" <td>897524.0</td>\n",
|
||
" <td>411029.0</td>\n",
|
||
" <td>1715900.0</td>\n",
|
||
" <td>532111.0</td>\n",
|
||
" <td>1.715900e+06</td>\n",
|
||
" <td>748231.000000</td>\n",
|
||
" <td>1.715900e+06</td>\n",
|
||
" <td>1.715900e+06</td>\n",
|
||
" <td>1.715900e+06</td>\n",
|
||
" <td>2.184924e+06</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>831.0</td>\n",
|
||
" <td>337799.000000</td>\n",
|
||
" <td>2184924.0</td>\n",
|
||
" <td>897524.0</td>\n",
|
||
" <td>3.167472e+06</td>\n",
|
||
" <td>1350.000000</td>\n",
|
||
" <td>46442.0</td>\n",
|
||
" <td>46442.0</td>\n",
|
||
" <td>3194467</td>\n",
|
||
" <td>3194464</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>mean</th>\n",
|
||
" <td>4.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>1.500802e+08</td>\n",
|
||
" <td>1459.691718</td>\n",
|
||
" <td>2.489274e+01</td>\n",
|
||
" <td>5.689277e+04</td>\n",
|
||
" <td>8.823625e+03</td>\n",
|
||
" <td>2.000152e+01</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>148.0</td>\n",
|
||
" <td>2499.803774</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.819241e+04</td>\n",
|
||
" <td>1413.825926</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>8 days 04:55:11.007770814</td>\n",
|
||
" <td>0 days 00:00:00.981176132</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>min</th>\n",
|
||
" <td>4.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>0.000000e+00</td>\n",
|
||
" <td>265.000000</td>\n",
|
||
" <td>2.000000e+01</td>\n",
|
||
" <td>0.000000e+00</td>\n",
|
||
" <td>0.000000e+00</td>\n",
|
||
" <td>2.000000e+01</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>148.0</td>\n",
|
||
" <td>1.000000</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>0.000000e+00</td>\n",
|
||
" <td>0.000000</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0 days 00:00:00</td>\n",
|
||
" <td>0 days 00:00:00</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>25%</th>\n",
|
||
" <td>4.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>1.000000e+00</td>\n",
|
||
" <td>1460.000000</td>\n",
|
||
" <td>2.000000e+01</td>\n",
|
||
" <td>1.500000e+01</td>\n",
|
||
" <td>5.010000e+02</td>\n",
|
||
" <td>2.000000e+01</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>148.0</td>\n",
|
||
" <td>38.000000</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>1.684083e+02</td>\n",
|
||
" <td>1.000000</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0 days 06:42:09.002197</td>\n",
|
||
" <td>0 days 00:00:00.000046</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>50%</th>\n",
|
||
" <td>4.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>1.000000e+00</td>\n",
|
||
" <td>1460.000000</td>\n",
|
||
" <td>2.400000e+01</td>\n",
|
||
" <td>1.027200e+04</td>\n",
|
||
" <td>1.024000e+03</td>\n",
|
||
" <td>2.000000e+01</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>148.0</td>\n",
|
||
" <td>69.000000</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>1.976696e+04</td>\n",
|
||
" <td>91.000000</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>9 days 12:24:21.461847</td>\n",
|
||
" <td>0 days 00:00:00.000473</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>75%</th>\n",
|
||
" <td>4.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>4.921175e+04</td>\n",
|
||
" <td>1460.000000</td>\n",
|
||
" <td>2.400000e+01</td>\n",
|
||
" <td>8.584425e+04</td>\n",
|
||
" <td>5.357000e+03</td>\n",
|
||
" <td>2.000000e+01</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>148.0</td>\n",
|
||
" <td>517.000000</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>5.255671e+04</td>\n",
|
||
" <td>259.750000</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>14 days 04:40:44.513388</td>\n",
|
||
" <td>0 days 00:00:00.071609</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>max</th>\n",
|
||
" <td>4.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>4.294967e+09</td>\n",
|
||
" <td>1460.000000</td>\n",
|
||
" <td>6.000000e+01</td>\n",
|
||
" <td>2.789480e+05</td>\n",
|
||
" <td>6.553500e+04</td>\n",
|
||
" <td>2.400000e+01</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>148.0</td>\n",
|
||
" <td>765073.000000</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>8.640009e+04</td>\n",
|
||
" <td>10354.000000</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>18 days 22:40:47.858279</td>\n",
|
||
" <td>7 days 21:15:30.172390</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>std</th>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>6.462904e+08</td>\n",
|
||
" <td>10.950512</td>\n",
|
||
" <td>5.737436e+00</td>\n",
|
||
" <td>8.001493e+04</td>\n",
|
||
" <td>1.786146e+04</td>\n",
|
||
" <td>7.799376e-02</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>24401.842325</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>2.870577e+04</td>\n",
|
||
" <td>3185.183304</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>6 days 21:06:14.202244411</td>\n",
|
||
" <td>0 days 00:08:27.290981739</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"<p>8 rows × 57 columns</p>\n",
|
||
"</div>"
|
||
]
|
||
},
|
||
"execution_count": 3,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"execution_count": 3
|
||
},
|
||
{
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2024-02-25T17:19:08.880696Z",
|
||
"start_time": "2024-02-25T17:19:08.861308Z"
|
||
}
|
||
},
|
||
"cell_type": "code",
|
||
"source": "merged_dataset.head()",
|
||
"id": "fc7ec5435335a843",
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
" tcp/tcp.flags/tcp.flags.reset arp/arp.proto.size \\\n",
|
||
"0 False NaN \n",
|
||
"1 False NaN \n",
|
||
"2 False NaN \n",
|
||
"3 False NaN \n",
|
||
"4 False NaN \n",
|
||
"\n",
|
||
" tcp/tcp.options/tcp.options.nop tcp/tcp.checksum.status \\\n",
|
||
"0 1.0 2.0 \n",
|
||
"1 1.0 2.0 \n",
|
||
"2 NaN 2.0 \n",
|
||
"3 NaN 2.0 \n",
|
||
"4 NaN 2.0 \n",
|
||
"\n",
|
||
" udp/udp.checksum.status tcp/tcp.nxtseq \\\n",
|
||
"0 NaN 1.0 \n",
|
||
"1 NaN 1.0 \n",
|
||
"2 NaN 70.0 \n",
|
||
"3 NaN 70.0 \n",
|
||
"4 NaN 70.0 \n",
|
||
"\n",
|
||
" tcp/tcp.options/tcp.options.mss/tcp.options.mss_val tcp/tcp.hdr_len \\\n",
|
||
"0 NaN 32.0 \n",
|
||
"1 NaN 32.0 \n",
|
||
"2 NaN 20.0 \n",
|
||
"3 NaN 20.0 \n",
|
||
"4 NaN 20.0 \n",
|
||
"\n",
|
||
" tcp/tcp.stream ip/ip.dst ... Farbeändern Helligkeitssteuern \\\n",
|
||
"0 0.0 192.168.1.201 ... NaN NaN \n",
|
||
"1 0.0 54.155.16.139 ... NaN NaN \n",
|
||
"2 1.0 76.223.105.139 ... NaN NaN \n",
|
||
"3 1.0 192.168.1.178 ... NaN NaN \n",
|
||
"4 1.0 76.223.105.139 ... NaN NaN \n",
|
||
"\n",
|
||
" Weiß/gelb Themen Auswahl Party Zeitschalter Timer Ein/Aus \\\n",
|
||
"0 NaN NaN NaN NaN NaN \n",
|
||
"1 NaN NaN NaN NaN NaN \n",
|
||
"2 NaN NaN NaN NaN NaN \n",
|
||
"3 NaN NaN NaN NaN NaN \n",
|
||
"4 NaN NaN NaN NaN NaN \n",
|
||
"\n",
|
||
" presence_absence illuminance_level Attack \n",
|
||
"0 NaN NaN NaN \n",
|
||
"1 NaN NaN NaN \n",
|
||
"2 NaN NaN NaN \n",
|
||
"3 NaN NaN NaN \n",
|
||
"4 NaN NaN NaN \n",
|
||
"\n",
|
||
"[5 rows x 101 columns]"
|
||
],
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>tcp/tcp.flags/tcp.flags.reset</th>\n",
|
||
" <th>arp/arp.proto.size</th>\n",
|
||
" <th>tcp/tcp.options/tcp.options.nop</th>\n",
|
||
" <th>tcp/tcp.checksum.status</th>\n",
|
||
" <th>udp/udp.checksum.status</th>\n",
|
||
" <th>tcp/tcp.nxtseq</th>\n",
|
||
" <th>tcp/tcp.options/tcp.options.mss/tcp.options.mss_val</th>\n",
|
||
" <th>tcp/tcp.hdr_len</th>\n",
|
||
" <th>tcp/tcp.stream</th>\n",
|
||
" <th>ip/ip.dst</th>\n",
|
||
" <th>...</th>\n",
|
||
" <th>Farbeändern</th>\n",
|
||
" <th>Helligkeitssteuern</th>\n",
|
||
" <th>Weiß/gelb</th>\n",
|
||
" <th>Themen Auswahl</th>\n",
|
||
" <th>Party</th>\n",
|
||
" <th>Zeitschalter</th>\n",
|
||
" <th>Timer Ein/Aus</th>\n",
|
||
" <th>presence_absence</th>\n",
|
||
" <th>illuminance_level</th>\n",
|
||
" <th>Attack</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>32.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>192.168.1.201</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>1</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>32.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>54.155.16.139</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>70.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>20.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>76.223.105.139</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>70.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>20.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>192.168.1.178</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>4</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>70.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>20.0</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>76.223.105.139</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"<p>5 rows × 101 columns</p>\n",
|
||
"</div>"
|
||
]
|
||
},
|
||
"execution_count": 4,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"execution_count": 4
|
||
},
|
||
{
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2024-02-25T17:19:10.617913Z",
|
||
"start_time": "2024-02-25T17:19:08.881777Z"
|
||
}
|
||
},
|
||
"cell_type": "code",
|
||
"source": [
|
||
"columns_to_ignore = [\n",
|
||
" 'Activity',\n",
|
||
" 'Attack',\n",
|
||
" 'Farbeändern',\n",
|
||
" 'Helligkeitssteuern',\n",
|
||
" 'Party',\n",
|
||
" 'Themen Auswahl',\n",
|
||
" 'Timer Ein/Aus',\n",
|
||
" 'Weiß/gelb',\n",
|
||
" 'Zeitschalter',\n",
|
||
" 'name',\n",
|
||
" 'presence_absence',\n",
|
||
" \"ip/ip.src\",\n",
|
||
" \"ip/ip.dst\",\n",
|
||
" \"eth/eth.src\",\n",
|
||
" \"eth/eth.dst\",\n",
|
||
" \"geninfo/timestamp\",\n",
|
||
" \"timedelta\",\n",
|
||
" \"timedelta_prev\",\n",
|
||
"]\n",
|
||
"\n",
|
||
"columns_to_predict = [\n",
|
||
" \"Attack\"\n",
|
||
"]\n",
|
||
"\n",
|
||
"list(merged_dataset.drop(columns=columns_to_ignore, inplace=False).dtypes)"
|
||
],
|
||
"id": "38bcc594f441b716",
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"[dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('O'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('O'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('O'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('O'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('O'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('O'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('O'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('O'),\n",
|
||
" dtype('O'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('bool'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('float64'),\n",
|
||
" dtype('O')]"
|
||
]
|
||
},
|
||
"execution_count": 5,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"execution_count": 5
|
||
},
|
||
{
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2024-02-25T17:23:55.928387Z",
|
||
"start_time": "2024-02-25T17:23:55.817968Z"
|
||
}
|
||
},
|
||
"cell_type": "code",
|
||
"source": "len(merged_dataset[merged_dataset[\"Attack\"] == 1])",
|
||
"id": "42735976a9d79bef",
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"581"
|
||
]
|
||
},
|
||
"execution_count": 8,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"execution_count": 8
|
||
},
|
||
{
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2024-02-25T17:19:19.505370Z",
|
||
"start_time": "2024-02-25T17:19:10.619581Z"
|
||
}
|
||
},
|
||
"cell_type": "code",
|
||
"source": [
|
||
"# prepare dataset for analysis\n",
|
||
"# drop all unecessary columns\n",
|
||
"dataset_to_use = merged_dataset.drop(columns=columns_to_ignore + columns_to_predict, inplace=False)\n",
|
||
"# also drop all object columns\n",
|
||
"dataset_to_use = dataset_to_use.select_dtypes(exclude=['object'])\n",
|
||
"\n",
|
||
"# fill NaN values with 0\n",
|
||
"dataset_to_use.fillna(0, inplace=True)\n",
|
||
"\n",
|
||
"dataset_y = merged_dataset[columns_to_predict[0]].fillna(0).astype(int)"
|
||
],
|
||
"id": "d2dc349a660c2e8a",
|
||
"outputs": [],
|
||
"execution_count": 6
|
||
},
|
||
{
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2024-02-25T17:13:29.479253Z",
|
||
"start_time": "2024-02-25T17:13:29.004170Z"
|
||
}
|
||
},
|
||
"cell_type": "code",
|
||
"source": "merged_dataset[columns_to_predict[0]].fillna(0).astype(int)",
|
||
"id": "fc6bba379e5fe8cd",
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"0 0\n",
|
||
"1 0\n",
|
||
"2 0\n",
|
||
"3 0\n",
|
||
"4 0\n",
|
||
" ..\n",
|
||
"1002184 0\n",
|
||
"1002185 0\n",
|
||
"1002186 0\n",
|
||
"1002187 1\n",
|
||
"1002188 0\n",
|
||
"Name: Attack, Length: 3194477, dtype: int64"
|
||
]
|
||
},
|
||
"execution_count": 29,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"execution_count": 29
|
||
},
|
||
{
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2024-02-25T17:23:17.250619Z",
|
||
"start_time": "2024-02-25T17:19:25.640575Z"
|
||
}
|
||
},
|
||
"cell_type": "code",
|
||
"source": [
|
||
"from imblearn.ensemble import BalancedRandomForestClassifier\n",
|
||
"from sklearn.ensemble import RandomForestClassifier\n",
|
||
"\n",
|
||
"from sklearn.model_selection import train_test_split\n",
|
||
"from sklearn.metrics import classification_report, confusion_matrix\n",
|
||
"\n",
|
||
"X_train, X_test, y_train, y_test = train_test_split(dataset_to_use,\n",
|
||
" dataset_y,\n",
|
||
" test_size=0.33,\n",
|
||
" random_state=42)\n",
|
||
"\n",
|
||
"brf_clf = RandomForestClassifier(n_estimators=100, random_state=42, class_weight='balanced')\n",
|
||
"brf_clf.fit(X_train, y_train)\n",
|
||
"\n",
|
||
"y_pred = brf_clf.predict(X_test)\n",
|
||
"\n",
|
||
"print(confusion_matrix(y_test, y_pred))\n"
|
||
],
|
||
"id": "908490c141f16963",
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[[1053969 4]\n",
|
||
" [ 36 169]]\n"
|
||
]
|
||
}
|
||
],
|
||
"execution_count": 7
|
||
},
|
||
{
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2024-02-25T14:32:09.084082Z",
|
||
"start_time": "2024-02-25T14:32:08.672777Z"
|
||
}
|
||
},
|
||
"cell_type": "code",
|
||
"source": [
|
||
"def get_ip_nodes(packet: dict | pd.Series, state=None):\n",
|
||
" if state is None:\n",
|
||
" state = set()\n",
|
||
"\n",
|
||
" if \"ip/ip.src\" in packet:\n",
|
||
" src = packet[\"ip/ip.src\"]\n",
|
||
" if src is not None and not pd.isna(src):\n",
|
||
" state.add(packet[\"ip/ip.src\"])\n",
|
||
" if \"ip/ip.dst\" in packet:\n",
|
||
" dst = packet[\"ip/ip.dst\"]\n",
|
||
" if dst is not None and not pd.isna(dst):\n",
|
||
" state.add(packet[\"ip/ip.dst\"])\n",
|
||
"\n",
|
||
" return state\n",
|
||
"\n",
|
||
"\n",
|
||
"limit = 10_000\n",
|
||
"state = process_packets(merged_dataset[:limit], [get_ip_nodes], verbose=True)\n",
|
||
"state"
|
||
],
|
||
"id": "bc220c885b374cb9",
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"100%|██████████| 10000/10000 [00:00<00:00, 531840.13it/s]\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"{'0.0.0.0',\n",
|
||
" '144.76.59.37',\n",
|
||
" '168.119.4.163',\n",
|
||
" '172.104.134.72',\n",
|
||
" '18.157.203.183',\n",
|
||
" '18.184.180.174',\n",
|
||
" '18.195.249.137',\n",
|
||
" '185.248.189.10',\n",
|
||
" '192.168.1.1',\n",
|
||
" '192.168.1.103',\n",
|
||
" '192.168.1.146',\n",
|
||
" '192.168.1.178',\n",
|
||
" '192.168.1.179',\n",
|
||
" '192.168.1.196',\n",
|
||
" '192.168.1.201',\n",
|
||
" '192.168.1.203',\n",
|
||
" '192.168.1.212',\n",
|
||
" '192.168.1.218',\n",
|
||
" '192.168.1.219',\n",
|
||
" '192.168.1.222',\n",
|
||
" '192.168.1.234',\n",
|
||
" '192.168.1.237',\n",
|
||
" '192.168.1.244',\n",
|
||
" '192.53.103.108',\n",
|
||
" '20.56.117.213',\n",
|
||
" '216.239.35.12',\n",
|
||
" '224.0.0.1',\n",
|
||
" '224.0.0.251',\n",
|
||
" '224.0.1.187',\n",
|
||
" '255.255.255.255',\n",
|
||
" '3.86.119.99',\n",
|
||
" '34.140.193.14',\n",
|
||
" '34.239.12.200',\n",
|
||
" '35.158.31.135',\n",
|
||
" '35.173.146.140',\n",
|
||
" '52.205.6.197',\n",
|
||
" '52.215.134.83',\n",
|
||
" '52.57.38.165',\n",
|
||
" '52.58.249.45',\n",
|
||
" '54.155.16.139',\n",
|
||
" '54.157.113.19',\n",
|
||
" '76.223.105.139',\n",
|
||
" '99.84.146.103',\n",
|
||
" '99.84.146.12',\n",
|
||
" '99.84.146.13',\n",
|
||
" '99.84.146.36'}"
|
||
]
|
||
},
|
||
"execution_count": 11,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"execution_count": 11
|
||
},
|
||
{
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2024-02-25T14:36:08.650369Z",
|
||
"start_time": "2024-02-25T14:36:08.092347Z"
|
||
}
|
||
},
|
||
"cell_type": "code",
|
||
"source": [
|
||
"def split_into_connections(packet: dict | pd.Series, state=None):\n",
|
||
" if \"ip/ip.src\" in packet and \"ip/ip.dst\" in packet:\n",
|
||
" src = packet[\"ip/ip.src\"]\n",
|
||
" dst = packet[\"ip/ip.dst\"]\n",
|
||
" else:\n",
|
||
" # discard packet\n",
|
||
" return state\n",
|
||
"\n",
|
||
" connection = (src, dst)\n",
|
||
" if state is None:\n",
|
||
" state = {}\n",
|
||
"\n",
|
||
" if connection not in state:\n",
|
||
" state[connection] = [packet]\n",
|
||
" else:\n",
|
||
" state[connection].append(packet)\n",
|
||
"\n",
|
||
" return state\n",
|
||
"\n",
|
||
"\n",
|
||
"processing_functions = [\n",
|
||
" split_into_connections\n",
|
||
"]\n",
|
||
"\n",
|
||
"limit = 10_000\n",
|
||
"state = process_packets(merged_dataset.iloc[:limit], processing_functions, verbose=True)"
|
||
],
|
||
"id": "a7df205bfbb31fb0",
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"100%|██████████| 10000/10000 [00:00<00:00, 341909.30it/s]\n"
|
||
]
|
||
}
|
||
],
|
||
"execution_count": 15
|
||
},
|
||
{
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2024-02-25T14:36:40.554999Z",
|
||
"start_time": "2024-02-25T14:36:40.514684Z"
|
||
}
|
||
},
|
||
"cell_type": "code",
|
||
"source": "pd.DataFrame(state[list(state.keys())[0]])",
|
||
"id": "1e2f7bb4352cd00a",
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
" tcp/tcp.flags/tcp.flags.reset arp/arp.proto.size \\\n",
|
||
"0 False NaN \n",
|
||
"1 False NaN \n",
|
||
"2 False NaN \n",
|
||
"3 False NaN \n",
|
||
"4 False NaN \n",
|
||
".. ... ... \n",
|
||
"253 False NaN \n",
|
||
"254 False NaN \n",
|
||
"255 False NaN \n",
|
||
"256 False NaN \n",
|
||
"257 False NaN \n",
|
||
"\n",
|
||
" tcp/tcp.options/tcp.options.nop tcp/tcp.checksum.status \\\n",
|
||
"0 1.0 2.0 \n",
|
||
"1 1.0 2.0 \n",
|
||
"2 1.0 2.0 \n",
|
||
"3 1.0 2.0 \n",
|
||
"4 1.0 2.0 \n",
|
||
".. ... ... \n",
|
||
"253 1.0 2.0 \n",
|
||
"254 1.0 2.0 \n",
|
||
"255 1.0 2.0 \n",
|
||
"256 1.0 2.0 \n",
|
||
"257 1.0 2.0 \n",
|
||
"\n",
|
||
" udp/udp.checksum.status tcp/tcp.nxtseq \\\n",
|
||
"0 NaN 1.0 \n",
|
||
"1 NaN 41.0 \n",
|
||
"2 NaN 88.0 \n",
|
||
"3 NaN 87.0 \n",
|
||
"4 NaN 127.0 \n",
|
||
".. ... ... \n",
|
||
"253 NaN 8987.0 \n",
|
||
"254 NaN 9027.0 \n",
|
||
"255 NaN 9026.0 \n",
|
||
"256 NaN 9066.0 \n",
|
||
"257 NaN 9065.0 \n",
|
||
"\n",
|
||
" tcp/tcp.options/tcp.options.mss/tcp.options.mss_val tcp/tcp.hdr_len \\\n",
|
||
"0 NaN 32.0 \n",
|
||
"1 NaN 32.0 \n",
|
||
"2 NaN 32.0 \n",
|
||
"3 NaN 32.0 \n",
|
||
"4 NaN 32.0 \n",
|
||
".. ... ... \n",
|
||
"253 NaN 32.0 \n",
|
||
"254 NaN 32.0 \n",
|
||
"255 NaN 32.0 \n",
|
||
"256 NaN 32.0 \n",
|
||
"257 NaN 32.0 \n",
|
||
"\n",
|
||
" tcp/tcp.stream ip/ip.dst ... Farbeändern Helligkeitssteuern \\\n",
|
||
"0 0.0 192.168.1.201 ... NaN NaN \n",
|
||
"1 0.0 192.168.1.201 ... NaN NaN \n",
|
||
"2 0.0 192.168.1.201 ... NaN NaN \n",
|
||
"3 0.0 192.168.1.201 ... NaN NaN \n",
|
||
"4 0.0 192.168.1.201 ... NaN NaN \n",
|
||
".. ... ... ... ... ... \n",
|
||
"253 0.0 192.168.1.201 ... NaN NaN \n",
|
||
"254 0.0 192.168.1.201 ... NaN NaN \n",
|
||
"255 0.0 192.168.1.201 ... NaN NaN \n",
|
||
"256 0.0 192.168.1.201 ... NaN NaN \n",
|
||
"257 0.0 192.168.1.201 ... NaN NaN \n",
|
||
"\n",
|
||
" Weiß/gelb Themen Auswahl Party Zeitschalter Timer Ein/Aus \\\n",
|
||
"0 NaN NaN NaN NaN NaN \n",
|
||
"1 NaN NaN NaN NaN NaN \n",
|
||
"2 NaN NaN NaN NaN NaN \n",
|
||
"3 NaN NaN NaN NaN NaN \n",
|
||
"4 NaN NaN NaN NaN NaN \n",
|
||
".. ... ... ... ... ... \n",
|
||
"253 NaN NaN NaN NaN NaN \n",
|
||
"254 NaN NaN NaN NaN NaN \n",
|
||
"255 NaN NaN NaN NaN NaN \n",
|
||
"256 NaN NaN NaN NaN NaN \n",
|
||
"257 NaN NaN NaN NaN NaN \n",
|
||
"\n",
|
||
" presence_absence illuminance_level Attack \n",
|
||
"0 NaN NaN NaN \n",
|
||
"1 NaN NaN NaN \n",
|
||
"2 NaN NaN NaN \n",
|
||
"3 NaN NaN NaN \n",
|
||
"4 NaN NaN NaN \n",
|
||
".. ... ... ... \n",
|
||
"253 NaN NaN NaN \n",
|
||
"254 NaN NaN NaN \n",
|
||
"255 NaN NaN NaN \n",
|
||
"256 NaN NaN NaN \n",
|
||
"257 NaN NaN NaN \n",
|
||
"\n",
|
||
"[258 rows x 101 columns]"
|
||
],
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>tcp/tcp.flags/tcp.flags.reset</th>\n",
|
||
" <th>arp/arp.proto.size</th>\n",
|
||
" <th>tcp/tcp.options/tcp.options.nop</th>\n",
|
||
" <th>tcp/tcp.checksum.status</th>\n",
|
||
" <th>udp/udp.checksum.status</th>\n",
|
||
" <th>tcp/tcp.nxtseq</th>\n",
|
||
" <th>tcp/tcp.options/tcp.options.mss/tcp.options.mss_val</th>\n",
|
||
" <th>tcp/tcp.hdr_len</th>\n",
|
||
" <th>tcp/tcp.stream</th>\n",
|
||
" <th>ip/ip.dst</th>\n",
|
||
" <th>...</th>\n",
|
||
" <th>Farbeändern</th>\n",
|
||
" <th>Helligkeitssteuern</th>\n",
|
||
" <th>Weiß/gelb</th>\n",
|
||
" <th>Themen Auswahl</th>\n",
|
||
" <th>Party</th>\n",
|
||
" <th>Zeitschalter</th>\n",
|
||
" <th>Timer Ein/Aus</th>\n",
|
||
" <th>presence_absence</th>\n",
|
||
" <th>illuminance_level</th>\n",
|
||
" <th>Attack</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>32.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>192.168.1.201</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>1</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>41.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>32.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>192.168.1.201</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>88.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>32.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>192.168.1.201</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>87.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>32.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>192.168.1.201</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>4</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>127.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>32.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>192.168.1.201</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>...</th>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>253</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>8987.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>32.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>192.168.1.201</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>254</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>9027.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>32.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>192.168.1.201</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>255</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>9026.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>32.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>192.168.1.201</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>256</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>9066.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>32.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>192.168.1.201</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>257</th>\n",
|
||
" <td>False</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>2.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>9065.0</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>32.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>192.168.1.201</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" <td>NaN</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"<p>258 rows × 101 columns</p>\n",
|
||
"</div>"
|
||
]
|
||
},
|
||
"execution_count": 18,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"execution_count": 18
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 2
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython2",
|
||
"version": "2.7.6"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|