{ "cells": [ { "metadata": { "ExecuteTime": { "end_time": "2024-02-27T12:58:21.532722137Z", "start_time": "2024-02-27T12:58:21.317772254Z" } }, "cell_type": "code", "source": [ "import json\n", "import os\n", "import time\n", "\n", "import pandas as pd\n", "import networkx as nx\n", "\n", "from kafka import KafkaConsumer, KafkaProducer, KafkaAdminClient\n", "import redis\n", "\n", "from graph_based_intrusion_detection.utils import constants\n", "from graph_based_intrusion_detection.utils.network_transport_utils import deserialize, serialize" ], "id": "62284ba9a54cad9b", "outputs": [], "execution_count": 1 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-27T12:58:21.533440468Z", "start_time": "2024-02-27T12:58:21.531812746Z" } }, "cell_type": "code", "source": [ "# constants\n", "devices_root = os.path.abspath('../../../')\n", "time_field = 'geninfo/timestamp'\n", "timedelta_field = \"timedelta\"\n" ], "id": "e2ac6b2bea4af750", "outputs": [], "execution_count": 2 }, { "cell_type": "code", "id": "initial_id", "metadata": { "collapsed": true, "ExecuteTime": { "end_time": "2024-02-27T12:58:24.623717709Z", "start_time": "2024-02-27T12:58:21.532068016Z" } }, "source": [ "# read merged dataset\n", "row_limit = None\n", "idle_dataset_path = os.path.join(devices_root, 'idle_merged.pickle')\n", "activity_dataset_path = os.path.join(devices_root, 'activity_merged.pickle')\n", "attack_dataset_path = os.path.join(devices_root, 'attack_merged.pickle')\n", "\n", "idle_dataset = pd.read_pickle(idle_dataset_path)\n", "activity_dataset = pd.read_pickle(activity_dataset_path)\n", "attack_dataset = pd.read_pickle(attack_dataset_path)\n", "\n", "merged_dataset_full = pd.concat([idle_dataset, activity_dataset, attack_dataset])\n", "merged_dataset = merged_dataset_full\n", "\n", "print(f\"Loaded merged dataset with {len(merged_dataset)} packets\")\n", "list(sorted(merged_dataset.columns))" ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loaded merged dataset with 3194477 packets\n" ] }, { "data": { "text/plain": "['Activity',\n 'Attack',\n 'Farbeändern',\n 'Helligkeitssteuern',\n 'Party',\n 'Themen Auswahl',\n 'Timer Ein/Aus',\n 'Weiß/gelb',\n 'Zeitschalter',\n 'arp/arp.hw.size',\n 'arp/arp.hw.type',\n 'arp/arp.opcode',\n 'arp/arp.proto.size',\n 'arp/arp.proto.type',\n 'eth/eth.dst',\n 'eth/eth.dst/eth.dst.ig',\n 'eth/eth.dst/eth.dst.oui',\n 'eth/eth.dst/eth.lg',\n 'eth/eth.src',\n 'eth/eth.src/eth.ig',\n 'eth/eth.src/eth.lg',\n 'eth/eth.src/eth.src.oui',\n 'eth/eth.type',\n 'frame/frame.encap_type',\n 'frame/frame.len',\n 'frame/frame.marked',\n 'frame/frame.number',\n 'frame/frame.offset_shift',\n 'frame/frame.time_delta',\n 'frame/frame.time_relative',\n 'geninfo/timestamp',\n 'icmp/data/data.len',\n 'icmp/icmp.checksum.status',\n 'icmp/icmp.code',\n 'icmp/icmp.ident',\n 'icmp/icmp.resp_in',\n 'icmp/icmp.resp_to',\n 'icmp/icmp.seq',\n 'icmp/icmp.seq_le',\n 'icmp/icmp.type',\n 'igmp/igmp.checksum.status',\n 'igmp/igmp.maddr',\n 'igmp/igmp.max_resp',\n 'igmp/igmp.type',\n 'illuminance_level',\n 'ip/<>/ip.options.routeralert/ip.opt.ra',\n 'ip/<>/ip.options.routeralert/ip.opt.type',\n 'ip/ip.checksum.status',\n 'ip/ip.dsfield',\n 'ip/ip.dsfield/ip.dsfield.dscp',\n 'ip/ip.dsfield/ip.dsfield.ecn',\n 'ip/ip.dst',\n 'ip/ip.flags',\n 'ip/ip.flags/ip.flags.df',\n 'ip/ip.flags/ip.flags.mf',\n 'ip/ip.flags/ip.flags.rb',\n 'ip/ip.frag_offset',\n 'ip/ip.hdr_len',\n 'ip/ip.id',\n 'ip/ip.len',\n 'ip/ip.proto',\n 'ip/ip.src',\n 'ip/ip.ttl',\n 'ip/ip.version',\n 'name',\n 'presence_absence',\n 'tcp/tcp.ack',\n 'tcp/tcp.analysis/tcp.analysis.bytes_in_flight',\n 'tcp/tcp.analysis/tcp.analysis.push_bytes_sent',\n 'tcp/tcp.checksum.status',\n 'tcp/tcp.completeness',\n 'tcp/tcp.dstport',\n 'tcp/tcp.flags/tcp.flags.ack',\n 'tcp/tcp.flags/tcp.flags.cwr',\n 'tcp/tcp.flags/tcp.flags.ecn',\n 'tcp/tcp.flags/tcp.flags.fin',\n 'tcp/tcp.flags/tcp.flags.ns',\n 'tcp/tcp.flags/tcp.flags.push',\n 'tcp/tcp.flags/tcp.flags.res',\n 'tcp/tcp.flags/tcp.flags.reset',\n 'tcp/tcp.flags/tcp.flags.syn',\n 'tcp/tcp.flags/tcp.flags.urg',\n 'tcp/tcp.hdr_len',\n 'tcp/tcp.len',\n 'tcp/tcp.nxtseq',\n 'tcp/tcp.options/tcp.options.mss/tcp.options.mss_val',\n 'tcp/tcp.options/tcp.options.nop',\n 'tcp/tcp.seq',\n 'tcp/tcp.srcport',\n 'tcp/tcp.stream',\n 'tcp/tcp.urgent_pointer',\n 'tcp/tcp.window_size',\n 'tcp/tcp.window_size_scalefactor',\n 'tcp/tcp.window_size_value',\n 'timedelta',\n 'timedelta_prev',\n 'udp/udp.checksum.status',\n 'udp/udp.dstport',\n 'udp/udp.length',\n 'udp/udp.srcport',\n 'udp/udp.stream']" }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 3 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-27T12:58:24.625494342Z", "start_time": "2024-02-27T12:58:24.624269320Z" } }, "cell_type": "code", "source": [ "merged_dataset[\"Attack\"]" ], "id": "eb6cc8d5fda6a592", "outputs": [ { "data": { "text/plain": "0 NaN\n1 NaN\n2 NaN\n3 NaN\n4 NaN\n ... \n1002184 False\n1002185 False\n1002186 False\n1002187 True\n1002188 False\nName: Attack, Length: 3194477, dtype: object" }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 4 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-27T12:58:24.737352398Z", "start_time": "2024-02-27T12:58:24.625896572Z" } }, "cell_type": "code", "source": [ "# create kafka producer\n", "producer = KafkaProducer(bootstrap_servers='localhost:9092')" ], "id": "1c916da24d18b2cb", "outputs": [], "execution_count": 5 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-27T12:58:25.101840964Z", "start_time": "2024-02-27T12:58:24.775561457Z" } }, "cell_type": "code", "source": [ "df_no_attacks = merged_dataset[merged_dataset[\"Attack\"] == 0]\n", "len(df_no_attacks)" ], "id": "f42cf1cb4efa5f0a", "outputs": [ { "data": { "text/plain": "1001608" }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 6 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-27T12:58:25.137020130Z", "start_time": "2024-02-27T12:58:25.134574937Z" } }, "cell_type": "code", "source": [ "df_with_attacks = merged_dataset[merged_dataset[\"Attack\"] > 0]\n", "len(df_with_attacks)" ], "id": "9a2be6344480c108", "outputs": [ { "data": { "text/plain": "581" }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 7 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-26T11:29:07.686861Z", "start_time": "2024-02-26T11:29:07.659441Z" } }, "cell_type": "code", "source": [ "df_to_use" ], "id": "c721f82aa7b67e1e", "outputs": [ { "data": { "text/plain": [ " tcp/tcp.flags/tcp.flags.reset arp/arp.proto.size \\\n", "1 False 4.0 \n", "624196 False NaN \n", "624197 False NaN \n", "624198 False NaN \n", "624199 False NaN \n", "... ... ... \n", "1001457 True NaN \n", "1001458 False 4.0 \n", "1001459 False 4.0 \n", "1001460 False NaN \n", "1002187 False NaN \n", "\n", " tcp/tcp.options/tcp.options.nop tcp/tcp.checksum.status \\\n", "1 NaN NaN \n", "624196 NaN 2.0 \n", "624197 NaN 2.0 \n", "624198 NaN 2.0 \n", "624199 NaN 2.0 \n", "... ... ... \n", "1001457 NaN 2.0 \n", "1001458 NaN NaN \n", "1001459 NaN NaN \n", "1001460 NaN 2.0 \n", "1002187 NaN NaN \n", "\n", " udp/udp.checksum.status tcp/tcp.nxtseq \\\n", "1 NaN NaN \n", "624196 NaN 354.0 \n", "624197 NaN 354.0 \n", "624198 NaN 353.0 \n", "624199 NaN 353.0 \n", "... ... ... \n", "1001457 NaN 1.0 \n", "1001458 NaN NaN \n", "1001459 NaN NaN \n", "1001460 NaN 4210.0 \n", "1002187 NaN NaN \n", "\n", " tcp/tcp.options/tcp.options.mss/tcp.options.mss_val tcp/tcp.hdr_len \\\n", "1 NaN NaN \n", "624196 NaN 20.0 \n", "624197 NaN 20.0 \n", "624198 NaN 20.0 \n", "624199 NaN 20.0 \n", "... ... ... \n", "1001457 NaN 20.0 \n", "1001458 NaN NaN \n", "1001459 NaN NaN \n", "1001460 NaN 20.0 \n", "1002187 NaN NaN \n", "\n", " tcp/tcp.stream ip/ip.dst ... Farbeändern Helligkeitssteuern \\\n", "1 NaN NaN ... NaN NaN \n", "624196 42.0 192.168.1.218 ... NaN NaN \n", "624197 42.0 192.168.1.218 ... NaN NaN \n", "624198 42.0 192.168.1.218 ... NaN NaN \n", "624199 42.0 192.168.1.218 ... NaN NaN \n", "... ... ... ... ... ... \n", "1001457 1002.0 192.168.1.229 ... NaN NaN \n", "1001458 NaN NaN ... NaN NaN \n", "1001459 NaN NaN ... NaN NaN \n", "1001460 0.0 76.223.105.139 ... NaN NaN \n", "1002187 NaN NaN ... NaN NaN \n", "\n", " Weiß/gelb Themen Auswahl Party Zeitschalter Timer Ein/Aus \\\n", "1 NaN NaN NaN NaN NaN \n", "624196 NaN NaN NaN NaN NaN \n", "624197 NaN NaN NaN NaN NaN \n", "624198 NaN NaN NaN NaN NaN \n", "624199 NaN NaN NaN NaN NaN \n", "... ... ... ... ... ... \n", "1001457 NaN NaN NaN NaN NaN \n", "1001458 NaN NaN NaN NaN NaN \n", "1001459 NaN NaN NaN NaN NaN \n", "1001460 NaN NaN NaN NaN NaN \n", "1002187 NaN NaN NaN NaN NaN \n", "\n", " presence_absence illuminance_level Attack \n", "1 NaN NaN True \n", "624196 NaN NaN True \n", "624197 NaN NaN True \n", "624198 NaN NaN True \n", "624199 NaN NaN True \n", "... ... ... ... \n", "1001457 NaN NaN True \n", "1001458 NaN NaN True \n", "1001459 NaN NaN True \n", "1001460 NaN NaN True \n", "1002187 NaN NaN True \n", "\n", "[581 rows x 101 columns]" ], "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
tcp/tcp.flags/tcp.flags.resetarp/arp.proto.sizetcp/tcp.options/tcp.options.noptcp/tcp.checksum.statusudp/udp.checksum.statustcp/tcp.nxtseqtcp/tcp.options/tcp.options.mss/tcp.options.mss_valtcp/tcp.hdr_lentcp/tcp.streamip/ip.dst...FarbeändernHelligkeitssteuernWeiß/gelbThemen AuswahlPartyZeitschalterTimer Ein/Auspresence_absenceilluminance_levelAttack
1False4.0NaNNaNNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNTrue
624196FalseNaNNaN2.0NaN354.0NaN20.042.0192.168.1.218...NaNNaNNaNNaNNaNNaNNaNNaNNaNTrue
624197FalseNaNNaN2.0NaN354.0NaN20.042.0192.168.1.218...NaNNaNNaNNaNNaNNaNNaNNaNNaNTrue
624198FalseNaNNaN2.0NaN353.0NaN20.042.0192.168.1.218...NaNNaNNaNNaNNaNNaNNaNNaNNaNTrue
624199FalseNaNNaN2.0NaN353.0NaN20.042.0192.168.1.218...NaNNaNNaNNaNNaNNaNNaNNaNNaNTrue
..................................................................
1001457TrueNaNNaN2.0NaN1.0NaN20.01002.0192.168.1.229...NaNNaNNaNNaNNaNNaNNaNNaNNaNTrue
1001458False4.0NaNNaNNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNTrue
1001459False4.0NaNNaNNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNTrue
1001460FalseNaNNaN2.0NaN4210.0NaN20.00.076.223.105.139...NaNNaNNaNNaNNaNNaNNaNNaNNaNTrue
1002187FalseNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNTrue
\n", "

581 rows × 101 columns

\n", "
" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 19 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-27T12:58:36.194341849Z", "start_time": "2024-02-27T12:58:36.119102012Z" } }, "cell_type": "code", "source": [ "# send packets to kafka\n", "offset = 000_000\n", "length = 1_000\n", "batch_size = 1000\n", "# for i in range(0, len(merged_dataset), batch_size):\n", "# batch = merged_dataset[i:i + batch_size]\n", "# for index, packet in batch.iterrows():\n", "# producer.send(constants.PACKET_TOPIC_NAME, value=kafka_utils.serialize(packet))\n", "# producer.flush()\n", "# print(f\"Sent batch {i} to {i + batch_size}\")\n", "\n", "df_to_use = df_with_attacks\n", "\n", "for index, packet in df_to_use.iloc[offset:offset + length].iterrows():\n", " producer.send(constants.PACKET_TOPIC_NAME, value=serialize(packet))" ], "id": "307ddb6624eb4c52", "outputs": [], "execution_count": 8 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T13:25:33.014970Z", "start_time": "2024-02-25T13:25:32.910090Z" } }, "cell_type": "code", "source": [ "# consume packets from kafka\n", "consumer = KafkaConsumer(constants.NODE_ANALYSIS_TOPIC,\n", " bootstrap_servers='localhost:9092',\n", " auto_offset_reset='earliest')\n" ], "id": "f856f4641b2722f2", "outputs": [], "execution_count": 33 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T13:05:08.556833Z", "start_time": "2024-02-25T13:05:08.445516Z" } }, "cell_type": "code", "source": [ "message = next(consumer)\n", "state = deserialize(message.value)\n", "state" ], "id": "e0e1ba15c59c5ed5", "outputs": [ { "data": { "text/plain": [ "{'node': '192.168.1.201',\n", " 'time': 1708865999.6744654,\n", " 'results': {'incoming_attack_probability': 0}}" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 25 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-23T17:04:38.040441Z", "start_time": "2024-02-23T17:04:37.347285Z" } }, "cell_type": "code", "source": [ "nx.draw(state[\"layer_3_graph\"])" ], "id": "a82367032278578d", "outputs": [ { "data": { "text/plain": [ "
" ], "image/png": "iVBORw0KGgoAAAANSUhEUgAAApQAAAHzCAYAAACe1o1DAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAdrklEQVR4nO3dXWxc53ng8efMTKiEjGpHJKh1djUQAkrahGGQABGkftBdbmshaQMBukiijX3loE2QRiiCOmihoNFaFwxaOEU/DHTR+qJoo1pJHXDLm3TtAoyhRWCFCDaJTFgi2UAeulHFkLQMmYypDGf2QqYtK/og887hDIe/35U9Izzvqy/ojzPnPZPV6/V6AADAL6jQ7A0AALC5CUoAAJIISgAAkghKAACSCEoAAJIISgAAkghKAACSCEoAAJIISgAAkghKAACSCEoAAJIISgAAkghKAACSCEoAAJIISgAAkghKAACSCEoAAJIISgAAkghKAACSCEoAAJIISgAAkghKAACSCEoAAJIISgAAkghKAACSCEoAAJIISgAAkghKAACSCEoAAJIISgAAkghKAACSCEoAAJIISgAAkghKAACSCEoAAJIISgAAkghKAACSlJq9Aa5bXK7GxfnFuFatRUepELu7u6Jrm98eAKD1KZYmmrp8NU6drcTYhdmoLCxF/Yb3sogo7+iMoX298eCBcuzZub1Z2wQAuKOsXq/X7/7DaKSZhaU4PnIuzkzPRbGQxUrt9r8Fq+8P9vXE8JGB2LWj84336vV6ZFm2EVsGALgtQbnBTo9X4sToRFRr9TuG5M2KhSxKhSwePdwfR/eXY/W3TVACAM0mKDfQ42NT8djTk8lz/uCBvXHsv+9pwI4AANI55b1BTo9XGhKTERFffWYyvj5eacgsAIBUgnIDzCwsxYnRiYbO/PLoRMwsLDV0JgDAL0JQboDjI+eiuo77JdeiWqvH8ZFzDZ0JAPCLEJQ5m7p8Nc5Mz63rAM5arNTqcWZ6LqZnrzZ0LgDAegnKnJ06W4liIZ+T2MVCFl97zr2UAEBzCcqcjV2YbfjVyVUrtXqMTc7mMhsAYK0EZY5eXa5GJeeDM5X5pVhcrua6BgDAnQjKHL04vxh5P+SzHhEX5xdzXgUA4PYEZY6uVWtttQ4AwK0Iyhx1lDbml3ej1gEAuBUlkqPd3V2R9zdtZ6+vAwDQLIIyR13bSlHe0ZnrGuXuzujaVsp1DQCAOxGUORva15vrcyiH9vbmMhsAYK0EZc4ePFDO9TmUDx0s5zIbAGCtBGXO9uzcHoN9PQ2/SlksZDHY1xN9vdsbOhcAYL0E5QYYPjIQpUIW0cCnUpYKWQwfGWjYPACAX5Sg3ADVVy5H8fvfjGjgme+Th/tjV84HfgAA1sLx4JxMT0/HN7/5zXjyySfjBz/4QURE7O95d8z+pwPJs794aF98cr97JwGA1pDV6/W8vx1wS3nyySdjeHg4nn/++ciyLFZ/ebMsi5dffjm+NflKnBidiGqtvq7DOsVCFqVCFicP94tJAKCl+Mi7wUZGRuL555+PiIgbW/1jH/tY3HPPPXF0fzn+9Qu/Hr/ynu6IiLse1ll9/1fe0x3/+oVfF5MAQMtxhbLBXn311XjggQfiueeee8vrp06dik996lNveW3q8tU4dbYSY5OzUZlfesuRnSyuP7R8aG9vPHSw7DQ3ANCyBGWDVavVGBgYiPPnz7/xWkdHR8zNzcX27bePwsXlalycX4xr1Vp0lAqxu7vLN+AAAJuCYmmgarUaH/jAB+L8+fPxqU99KqampmJ8fDx++7d/+44xGXH9axr7333PBu0UAKBxBGWDVKvVeP/73x8XLlyIT3/60/HEE0/EK6+8EseOHYvf+73fa/b2AABy4yPvBqhWq9Hf3x+Tk5PxO7/zO/E3f/M3zd4SAMCGcco70bVr1+K9731vTE5Oxmc/+1kxCQBsOYJynZ544on48Ic/HJcuXXojJqenp+Nzn/tc/PVf/3WztwcAsOF85L1O73vf++KFF16I97znPXHt2rV46aWX4tixY/GXf/mXzd4aAEBTCMp1eOGFF+J973vfW15zzyQAsNVt+VPe63n+4ze+8Y0oFouxsrISEde/TvHZZ5+N2dnZ6O3t3chtAwC0jC15hfKNb6i5MBuVhVt8Q82Ozhja1xsPHijHnp3Xnx9Zr9djz5498W//9m8/N+/o0aPx5JNPbszmAQBazJYKypmFpTg+ci7OTM9FsZDFSu32P/XV9wf7emL4yEBcmn4+Dhw4EBERhUIharVadHd3x8c//vE4duzYz30UDgCwVWyZoDw9XokToxNRrdXvGJI3KxayKBWy+On//fuY+fbp6OzsjIcffjg+/vGPx6/+6q9GsVjMcdcAAK1vSwTl42NT8djTkwkT6hGRxYeKM/HNk78bhYKnLQEArGr7Mjo9XkmMyYjrd1ZG/L+VXfFP33spfVMAAG2krYNyZmEpToxONHTml0cnYmZhqaEzAQA2s7YOyuMj56K6jvsl16Jaq8fxkXMNnQkAsJm1bVBOXb4aZ6bn1nUAZy1WavU4Mz0X07NXGzoXAGCzatugPHW2EsVClsvsYiGLrz1XyWU2AMBm07ZBOXZhtuFXJ1et1OoxNjmby2wAgM2mLYPy1eVqVHI+OFOZX4rF5WquawAAbAZtGZQvzi9G3g/XrEfExfnFnFcBAGh9bRmU16q1tloHAKCVtWVQdpQ25qe1UesAALSytiyi3d1dkc/57jdlr68DALDVtWVQdm0rRXlHZ65rlLs7o2tbKdc1AAA2g7YMyoiIoX29uT6Hcmhvby6zAQA2m7YNygcPlHN9DuVDB8u5zAYA2GzaNij37Nweg309Db9KWSxkMdjXE3292xs6FwBgs2rboIyIGD4yEKUGB2WpkMXwkYGGzgQA2MzaOih37eiMRw/3N3TmycP9sSvnAz8AAJtJWwdlRMTR/eV45NDehsz64qF98cn97p0EALhRVq/X8/6WwpZwerwSJ0Ynolqrr+uwTrGQRamQxcnD/WISAOAWtkxQRkTMLCzF8ZFzcWZ6LoqF7I5hufr+YF9PDB8Z8DE3AMBtbKmgXDV1+WqcOluJscnZqMwvxY2/AFlcf2j50N7eeOhg2WluAIC72JJBeaPF5WpcnF+Ma9VadJQKsbu7yzfgAACsw5YPSgAA0rT9KW8AAPIlKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASCIoAQBIIigBAEgiKAEASFJq9gYAAFrB4nI1Ls4vxrVqLTpKhdjd3RVd26TSWvhVAgC2rKnLV+PU2UqMXZiNysJS1G94L4uI8o7OGNrXGw8eKMeendubtc2Wl9Xr9frdfxgAQPuYWViK4yPn4sz0XBQLWazUbp9Dq+8P9vXE8JGB2LWjcwN3ujkISgBgSzk9XokToxNRrdXvGJI3KxayKBWyePRwfxzdX85xh5uPoAQAtozHx6bisacnk+c8cmhvfH5oTwN21B6c8gYAtoTT45WGxGRExGNPT8bXxysNmdUOBCUA0PZmFpbixOhEQ2d+eXQiZhaWGjpzsxKUAEDbOz5yLqrruF9yLaq1ehwfOdfQmZuVoAQA2trU5atxZnpuXQdw1mKlVo8z03MxPXu1oXM3I0EJALS1U2crUSxkucwuFrL42nPupRSUAEBbG7sw2/Crk6tWavUYm5zNZfZmIigBgLb16nI1KjkfnKnML8XicjXXNVqdoAQA2taL84uR9wO36xFxcX4x51Vam6AEANrWtWqtrdZpVYISAGhbHaWNSZ2NWqdVbe2fPQDQ1nZ3d0U+57vflL2+zlYmKAGAttW1rRTlHZ25rlHu7oyubaVc12h1ghIAaGtD+3pzfQ7l0N7eXGZvJoISAGhrDx4o5/ocyocOlnOZvZkISgCgre3ZuT0G+3oafpWyWMhisK8n+nq3N3TuZiQoAYC2N3xkIEoNDspSIYvhIwMNnblZCUoAoO3t2tEZjx7ub+jMk4f7Y1fOB342C0EJAGwJR/eX45FDexsy64uH9sUn97t3clVWr9fz/kYiAICWcXq8EidGJ6Jaq6/rsE6xkEWpkMXJw/1i8iaCEgDYcmYWluL4yLk4Mz0XxUJ2x7BcfX+wryeGjwz4mPsWBCUAsGVNXb4ap85WYmxyNirzS3FzFP1s4cdxaOA/x5c+Meg09x0ISgCAiFhcrsbF+cW4Vq1FR6kQvZ1Z9L7rnrj33nvj8uXL0dHR0ewttiyHcgAA4vrXNPa/+574UPld0f/ue2L6hYmIiLhy5Up85jOfCdfgbk9QAgDcwnPPPffGf//d3/1dfOUrX2niblqboAQAuIXvfOc7kWVvPgz9S1/6Upw+fbqJO2pd7qEEALiF++67L/7jP/7jLa+97W1vi+9+97vxwQ9+sDmbalGuUAIA3OTf//3ffy4mIyJ6enripz/9aRN21NpKzd4AAECrmZ2djYiInTt3xtve9rZ46aWX4vvf/3584AMfeMvH4FznCiUAwE0+9KEPxSuvvBKXLl2KP/3TP42IiGeffVZM3oZ7KAEA7qBarUZHR0fcf//98e1vf7vZ22lJghIA4C7uu+++eO211+Lll19u9lZako+8AQDu4uDBg3HlypW4cuVKs7fSkgQlAMBdHD16NCIivvCFL8SxY8fiox/9aLz66qtN3lXr8JE3AMBtXLx4Mf7sz/4snnnmmTh//nxERGRZFvV6PX7yk59ET09Pk3fYGjw2CADgNkZHR+Ov/uqv3vJavV6P3bt3i8kb+MgbAOA2Pve5z8UDDzwQxWLxjdeKxWIMDg42cVetR1ACANxGqVSKp556Kvbu3RuFwvVsWllZiV/+5V9u8s5ai6AEALiDX/qlX4p/+Zd/iR07drzxmqB8K4dyAADW4Hvf+158+MMfjoiIn/3sZ1EqOYqySlACAKzRJz7xiXjqqadi9uVX4vJiLa5Va9FRKsTu7q7o2rZ1A1NQAgCswdTlq/H4//lBfPM756P0rvve8l4WEeUdnTG0rzcePFCOPTu3N2eTTSIoAQDuYGZhKY6PnIsz03NRLGSxUrt9Oq2+P9jXE8NHBmLXjs4N3GnzCEoAgNs4PV6JE6MTUa3V7xiSNysWsigVsnj0cH8c3V/OcYetQVACANzC42NT8djTk8lzHjm0Nz4/tKcBO2pdHhsEAHCT0+OVhsRkRMRjT0/G18crDZnVqgQlAMANZhaW4sToRENnfnl0ImYWlho6s5UISgCAGxwfORfVddwvuRbVWj2Oj5xr6MxWIigBAF43dflqnJmeW9cBnLVYqdXjzPRcTM9ebejcViEoAQBed+psJYqFLJfZxUIWX3uuPe+lFJQAAK8buzDb8KuTq1Zq9RibnM1ldrMJSgCAiHh1uRqVnA/OVOaXYnG5musazSAoAQAi4sX5xcj74dz1iLg4v5jzKhtPUAIARMS1aq2t1tlIghIAICI6ShuTRRu1zkZqv58RAMAvYHd3V+RzvvtN2evrtBtBCQAQEV3bSlHe0ZnrGuXuzujaVsp1jWYQlAAArxva15vrcyiH9vbmMrvZBCUAwOsePFDO9TmUDx0s5zK72QQlAMDr9uzcHoN9PQ2/SlksZDHY1xN9vdsbOrdVCEoAgBsMHxmIUoODslTIYvjIQENnthJBCQBwg107OuPRw/0NnXnycH/syvnATzMJSgCAmxzdX45HDu1tyKwvHtoXn9zfnvdOrsrq9Xre3zIEALApnR6vxInRiajW6us6rFMsZFEqZHHycH/bx2SEoAQAuKOZhaU4PnIuzkzPRbGQ3TEsV98f7OuJ4SMDbf0x940EJQDAGkxdvhqnzlZibHI2KvNLcWNAZXH9oeVDe3vjoYPltj3NfTuCEgBgnRaXq3FxfjGuVWvRUSrE7u6utvwGnLUSlAAAJHHKGwCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJKVmbwA2q8XlalycX4xr1Vp0lAqxu7srurb5KwXA1uNfP1iHqctX49TZSoxdmI3KwlLUb3gvi4jyjs4Y2tcbDx4ox56d25u1TQDYUFm9Xq/f/YfB1jazsBTHR87Fmem5KBayWKnd/q/N6vuDfT0xfGQgdu3o3MCdAsDGE5RwF6fHK3FidCKqtfodQ/JmxUIWpUIWjx7uj6P7yznuEACaS1DCHTw+NhWPPT2ZPOeRQ3vj80N7GrAjAGg9TnnDbZwerzQkJiMiHnt6Mr4+XmnILABoNYISbmFmYSlOjE40dOaXRydiZmGpoTMBoBUISriF4yPnorqO+yXXolqrx/GRcw2dCQCtQFDCTaYuX40z03PrOoCzFiu1epyZnovp2asNnQsAzSYo4SanzlaiWMhymV0sZPG159xLCUB7EZRwk7ELsw2/OrlqpVaPscnZXGYDQLMISrjBq8vVqOR8cKYyvxSLy9Vc1wCAjSQo4QYvzi9G3g9mrUfExfnFnFcBgI0jKOEG16q1tloHADaCoIQbdJQ25q/ERq0DABvBv2pwg93dXZHP+e43Za+vAwDtQlDCDbq2laK8ozPXNcrdndG1rZTrGgCwkQQl3GRoX2+uz6Ec2tuby2wAaBZBCTd58EA51+dQPnSwnMtsAGgWQQk32bNzewz29TT8KmWxkMVgX0/09W5v6FwAaDZBCbcwfGQgSg0OylIhi+EjAw2dCQCtQFDCLeza0RmPHu5v6MyTh/tjV84HfgCgGQQl3MbR/eV45NDehsz64qF98cn97p0EoD1l9Xo972+ag03t9HglToxOxM+qK1Fbx1Mqi4UsSoUsTh7uF5MAtDVBCWvwo9lX4v4/+F9R+i/vj2Ihu+Mp8NX3B/t6YvjIgI+5AWh7ghLu4rXXXosDBw7ED3/4w3j4C1+KXf/tf8TY5GxU5pfixr88WVx/aPnQ3t546GDZaW4Atgxf1wF3cOXKlfjYxz4WP/zhDyMi4sB/LcfvHu6P/xn9sbhcjYvzi3GtWouOUiF2d3f5BhwAtiT/+sFtvPTSS/HAAw/E1NRURERkWRZzc3NvvN+1rRT9776nWdsDgJYhKOEWXnjhhfiN3/iNmJ2djZWVlYiIKBaLcfny5SbvDABaj6CEW/iLv/iLuHTpUmTZm6e6a7VazM7ONnFXANCaPIcSbuGrX/1qPPHEE7Fr166IiCgUClGr1eLSpUtN3hkAtB5BCbfQ1dUVn/70p6OjoyOKxWJ89rOfjXe+853N3hYAtCSPDYLbePHFF2P37t3xm7/5m/HMM8/Ea6+9FrVaLTo7PVcSAG7kHkq4jT/8wz+MiIg/+ZM/iYiIt7/97c3cDgC0LFco4RZqtVp0dXXFvffe675JALgL91DCLfzDP/xDvPbaa/GZz3ym2VsBgJbnCiXcwnvf+96YmpqKpaWl6OjoaPZ2AKCluUIJN/nxj38c58+fj1/7tV8TkwCwBoISbvJHf/RHERHxla98pck7AYDNwUfeEBEvv/xy3HvvvZFlWbzzne+Mzs5O34oDAGvksUFseVeuXIne3t4ol8tx8ODBWFxcjGPHjjV7WwCwafjImy2vVCpFtVqNH/3oR/GP//iPERFx/vz5eOaZZ8IFfAC4Ox95s+XV6/V4xzveEcvLy2+8tvrd3d/61rfiIx/5SBN3BwCtzxVKtrwsy6K7u/str9Xr9fit3/qtuP/++5u0KwDYPAQlRMTOnTvf8v8PP/xw/PM//7Pv7QaANRCUENc/4l71x3/8x/G3f/u3USo5swYAa+FfTLaUxeVqXJxfjGvVWnSUCrG7uyu6tpViZmYmIiL+/M//PH7/93+/ybsEgM3FoRza3tTlq3HqbCXGLsxGZWEpbvwDn0XErne9I6ae/d/x/ndciWee+vtmbRMANi1BSduaWViK4yPn4sz0XBQLWazU7vBHvV6LyAox2NcTw0cGYtcO904CwFoJStrS6fFKnBidiGqtfueQvEmxkEWpkMWjh/vj6P5yjjsEgPYhKGk7j49NxWNPTybPeeTQ3vj80J4G7AgA2ptT3rSV0+OVhsRkRMRjT0/G18crDZkFAO1MUNI2ZhaW4sToRENnfnl0ImYWlho6EwDajaCkbRwfORfVddwvuRbVWj2Oj5xr6EwAaDeCkrYwdflqnJmeW9cBnLVYqdXjzPRcTM9ebehcAGgngpK2cOpsJYqFLJfZxUIWX3vOvZQAcDuCkrYwdmG24VcnV63U6jE2OZvLbABoB4KSTe/V5WpUcj44U5lfisXlaq5rAMBmJSjZ9F6cX4y8H6Zaj4iL84s5rwIAm5OgZNO7Vq211ToAsNkISja9jtLG/DHeqHUAYLPxLySb3u7ursjnfPebstfXAQB+nqBk0+vaVoryjs5c1yh3d0bXtlKuawDAZiUoaQtD+3pzfQ7l0N7eXGYDQDsQlLSFBw+Uc30O5UMHy7nMBoB2IChpC3t2bo/Bvp6GX6UsFrIY7OuJvt7tDZ0LAO1EUNI2ho8MRKnBQVkqZDF8ZKChMwGg3QhK2sauHZ3x6OH+hs48ebg/duV84AcANjtBSVs5ur8cjxza25BZXzy0Lz65372TAHA3Wb1ez/tb62DDnR6vxInRiajW6us6rFMsZFEqZHHycL+YBIA1EpS0rZmFpTg+ci7OTM9FsZDdMSxX3x/s64nhIwM+5gaAdRCUtL2py1fj1NlKjE3ORmV+KW78A5/F9YeWD+3tjYcOlp3mBoBfgKBkS1lcrsbF+cW4Vq1FR6kQu7u7fAMOACQSlAAAJHHKGwCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJIISAIAkghIAgCSCEgCAJP8fzTmRlkq2b7wAAAAASUVORK5CYII=" }, "metadata": {}, "output_type": "display_data" } ], "execution_count": 23 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T13:28:19.034216Z", "start_time": "2024-02-25T13:28:10.962535Z" } }, "cell_type": "code", "source": [ "# consume whole topic until empty\n", "counter = 0\n", "for message in consumer:\n", " counter += 1\n", " print(\"Consumed message: \", counter)" ], "id": "f25e8dbee091fc8", "outputs": [ { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", "\u001B[0;31mKeyboardInterrupt\u001B[0m Traceback (most recent call last)", "Cell \u001B[0;32mIn[35], line 3\u001B[0m\n\u001B[1;32m 1\u001B[0m \u001B[38;5;66;03m# consume whole topic until empty\u001B[39;00m\n\u001B[1;32m 2\u001B[0m counter \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m\n\u001B[0;32m----> 3\u001B[0m \u001B[38;5;28;43;01mfor\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mmessage\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;129;43;01min\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mconsumer\u001B[49m\u001B[43m:\u001B[49m\n\u001B[1;32m 4\u001B[0m \u001B[43m \u001B[49m\u001B[43mcounter\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m \u001B[49m\u001B[38;5;241;43m1\u001B[39;49m\n\u001B[1;32m 5\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;28;43mprint\u001B[39;49m\u001B[43m(\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mConsumed message: \u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcounter\u001B[49m\u001B[43m)\u001B[49m\n", "File \u001B[0;32m~/projects/it-security-praktikum/venv/lib/python3.11/site-packages/kafka/consumer/group.py:1193\u001B[0m, in \u001B[0;36mKafkaConsumer.__next__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 1191\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mnext_v1()\n\u001B[1;32m 1192\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m-> 1193\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mnext_v2\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n", "File \u001B[0;32m~/projects/it-security-praktikum/venv/lib/python3.11/site-packages/kafka/consumer/group.py:1201\u001B[0m, in \u001B[0;36mKafkaConsumer.next_v2\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 1199\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_iterator \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_message_generator_v2()\n\u001B[1;32m 1200\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m-> 1201\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mnext\u001B[39m(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_iterator)\n\u001B[1;32m 1202\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mStopIteration\u001B[39;00m:\n\u001B[1;32m 1203\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_iterator \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m\n", "File \u001B[0;32m~/projects/it-security-praktikum/venv/lib/python3.11/site-packages/kafka/consumer/group.py:1116\u001B[0m, in \u001B[0;36mKafkaConsumer._message_generator_v2\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 1114\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_message_generator_v2\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[1;32m 1115\u001B[0m timeout_ms \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m1000\u001B[39m \u001B[38;5;241m*\u001B[39m (\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_consumer_timeout \u001B[38;5;241m-\u001B[39m time\u001B[38;5;241m.\u001B[39mtime())\n\u001B[0;32m-> 1116\u001B[0m record_map \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpoll\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtimeout_ms\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtimeout_ms\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mupdate_offsets\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m)\u001B[49m\n\u001B[1;32m 1117\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m tp, records \u001B[38;5;129;01min\u001B[39;00m six\u001B[38;5;241m.\u001B[39miteritems(record_map):\n\u001B[1;32m 1118\u001B[0m \u001B[38;5;66;03m# Generators are stateful, and it is possible that the tp / records\u001B[39;00m\n\u001B[1;32m 1119\u001B[0m \u001B[38;5;66;03m# here may become stale during iteration -- i.e., we seek to a\u001B[39;00m\n\u001B[1;32m 1120\u001B[0m \u001B[38;5;66;03m# different offset, pause consumption, or lose assignment.\u001B[39;00m\n\u001B[1;32m 1121\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m record \u001B[38;5;129;01min\u001B[39;00m records:\n\u001B[1;32m 1122\u001B[0m \u001B[38;5;66;03m# is_fetchable(tp) should handle assignment changes and offset\u001B[39;00m\n\u001B[1;32m 1123\u001B[0m \u001B[38;5;66;03m# resets; for all other changes (e.g., seeks) we'll rely on the\u001B[39;00m\n\u001B[1;32m 1124\u001B[0m \u001B[38;5;66;03m# outer function destroying the existing iterator/generator\u001B[39;00m\n\u001B[1;32m 1125\u001B[0m \u001B[38;5;66;03m# via self._iterator = None\u001B[39;00m\n", "File \u001B[0;32m~/projects/it-security-praktikum/venv/lib/python3.11/site-packages/kafka/consumer/group.py:655\u001B[0m, in \u001B[0;36mKafkaConsumer.poll\u001B[0;34m(self, timeout_ms, max_records, update_offsets)\u001B[0m\n\u001B[1;32m 653\u001B[0m remaining \u001B[38;5;241m=\u001B[39m timeout_ms\n\u001B[1;32m 654\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[0;32m--> 655\u001B[0m records \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_poll_once\u001B[49m\u001B[43m(\u001B[49m\u001B[43mremaining\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmax_records\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mupdate_offsets\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mupdate_offsets\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 656\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m records:\n\u001B[1;32m 657\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m records\n", "File \u001B[0;32m~/projects/it-security-praktikum/venv/lib/python3.11/site-packages/kafka/consumer/group.py:702\u001B[0m, in \u001B[0;36mKafkaConsumer._poll_once\u001B[0;34m(self, timeout_ms, max_records, update_offsets)\u001B[0m\n\u001B[1;32m 699\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_client\u001B[38;5;241m.\u001B[39mpoll(timeout_ms\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m0\u001B[39m)\n\u001B[1;32m 701\u001B[0m timeout_ms \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mmin\u001B[39m(timeout_ms, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_coordinator\u001B[38;5;241m.\u001B[39mtime_to_next_poll() \u001B[38;5;241m*\u001B[39m \u001B[38;5;241m1000\u001B[39m)\n\u001B[0;32m--> 702\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_client\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpoll\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtimeout_ms\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtimeout_ms\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 703\u001B[0m \u001B[38;5;66;03m# after the long poll, we should check whether the group needs to rebalance\u001B[39;00m\n\u001B[1;32m 704\u001B[0m \u001B[38;5;66;03m# prior to returning data so that the group can stabilize faster\u001B[39;00m\n\u001B[1;32m 705\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_coordinator\u001B[38;5;241m.\u001B[39mneed_rejoin():\n", "File \u001B[0;32m~/projects/it-security-praktikum/venv/lib/python3.11/site-packages/kafka/client_async.py:602\u001B[0m, in \u001B[0;36mKafkaClient.poll\u001B[0;34m(self, timeout_ms, future)\u001B[0m\n\u001B[1;32m 599\u001B[0m timeout \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mmin\u001B[39m(timeout, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mconfig[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mretry_backoff_ms\u001B[39m\u001B[38;5;124m'\u001B[39m])\n\u001B[1;32m 600\u001B[0m timeout \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mmax\u001B[39m(\u001B[38;5;241m0\u001B[39m, timeout) \u001B[38;5;66;03m# avoid negative timeouts\u001B[39;00m\n\u001B[0;32m--> 602\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_poll\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtimeout\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m/\u001B[39;49m\u001B[43m \u001B[49m\u001B[38;5;241;43m1000\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[1;32m 604\u001B[0m \u001B[38;5;66;03m# called without the lock to avoid deadlock potential\u001B[39;00m\n\u001B[1;32m 605\u001B[0m \u001B[38;5;66;03m# if handlers need to acquire locks\u001B[39;00m\n\u001B[1;32m 606\u001B[0m responses\u001B[38;5;241m.\u001B[39mextend(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_fire_pending_completed_requests())\n", "File \u001B[0;32m~/projects/it-security-praktikum/venv/lib/python3.11/site-packages/kafka/client_async.py:634\u001B[0m, in \u001B[0;36mKafkaClient._poll\u001B[0;34m(self, timeout)\u001B[0m\n\u001B[1;32m 631\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_register_send_sockets()\n\u001B[1;32m 633\u001B[0m start_select \u001B[38;5;241m=\u001B[39m time\u001B[38;5;241m.\u001B[39mtime()\n\u001B[0;32m--> 634\u001B[0m ready \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_selector\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mselect\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtimeout\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 635\u001B[0m end_select \u001B[38;5;241m=\u001B[39m time\u001B[38;5;241m.\u001B[39mtime()\n\u001B[1;32m 636\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_sensors:\n", "File \u001B[0;32m/usr/lib/python3.11/selectors.py:468\u001B[0m, in \u001B[0;36mEpollSelector.select\u001B[0;34m(self, timeout)\u001B[0m\n\u001B[1;32m 466\u001B[0m ready \u001B[38;5;241m=\u001B[39m []\n\u001B[1;32m 467\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 468\u001B[0m fd_event_list \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_selector\u001B[38;5;241m.\u001B[39mpoll(timeout, max_ev)\n\u001B[1;32m 469\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mInterruptedError\u001B[39;00m:\n\u001B[1;32m 470\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m ready\n", "\u001B[0;31mKeyboardInterrupt\u001B[0m: " ] } ], "execution_count": 35 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-23T16:06:14.532212Z", "start_time": "2024-02-23T16:06:09.157947Z" } }, "cell_type": "code", "source": [ "while True:\n", " for message in consumer:\n", " packet = json.loads(message.value)\n", " print(\"Received packet: \", packet[\"timedelta\"])\n", " time.sleep(1)" ], "id": "f57c14b5a04f7ea", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Received packet: 3621\n", "Received packet: 3621\n", "Received packet: 3831\n", "Received packet: 3831\n", "Received packet: 3837\n", "Received packet: 3837\n" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", "\u001B[0;31mKeyboardInterrupt\u001B[0m Traceback (most recent call last)", "Cell \u001B[0;32mIn[50], line 6\u001B[0m\n\u001B[1;32m 4\u001B[0m packet \u001B[38;5;241m=\u001B[39m json\u001B[38;5;241m.\u001B[39mloads(message\u001B[38;5;241m.\u001B[39mvalue)\n\u001B[1;32m 5\u001B[0m \u001B[38;5;28mprint\u001B[39m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mReceived packet: \u001B[39m\u001B[38;5;124m\"\u001B[39m, packet[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtimedelta\u001B[39m\u001B[38;5;124m\"\u001B[39m])\n\u001B[0;32m----> 6\u001B[0m \u001B[43mtime\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msleep\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m1\u001B[39;49m\u001B[43m)\u001B[49m\n", "\u001B[0;31mKeyboardInterrupt\u001B[0m: " ] } ], "execution_count": 50 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T13:05:16.406426Z", "start_time": "2024-02-25T13:05:16.404128Z" } }, "cell_type": "code", "source": [ "# connect to redis\n", "redis_client = redis.StrictRedis(host=\"localhost\", port=6379, db=0)" ], "id": "be20e147b3839d33", "outputs": [], "execution_count": 26 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T12:53:29.562348Z", "start_time": "2024-02-25T12:53:29.559317Z" } }, "cell_type": "code", "source": [ "redis_client.delete(constants.NETWORK_STATE_KEY)" ], "id": "578a6af45df433d3", "outputs": [ { "data": { "text/plain": [ "1" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 14 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T13:05:20.457325Z", "start_time": "2024-02-25T13:05:20.453392Z" } }, "cell_type": "code", "source": [ "\n", "redis_client.get(constants.NETWORK_STATE_KEY)\n", "\n", "network_state = deserialize(redis_client.get(constants.NETWORK_STATE_KEY))\n", "network_state[\"layer_3_graph\"].nodes" ], "id": "eb058f92760340ed", "outputs": [ { "data": { "text/plain": [ "NodeView(('192.168.1.201', '99.84.146.36', '54.155.16.139', '192.168.1.234', '255.255.255.255'))" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 27 }, { "metadata": { "ExecuteTime": { "end_time": "2024-02-25T11:58:44.019607Z", "start_time": "2024-02-25T11:58:44.010703Z" } }, "cell_type": "code", "source": [ "node_registry_raw = redis_client.get(constants.NODE_REGISTRY_KEY)\n", "node_registry = deserialize(node_registry_raw)\n", "node_registry" ], "id": "b9e9018bdbd44da6", "outputs": [ { "data": { "text/plain": [ "{'54.155.16.139': {'last_updated': 1708862316.1514397,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.201': {'last_updated': 1708862316.1773772,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.178': {'last_updated': 1708862316.1915157,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '76.223.105.139': {'last_updated': 1708862316.215673,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '20.56.117.213': {'last_updated': 1708862316.2553146,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.196': {'last_updated': 1708862316.287682,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.244': {'last_updated': 1708862316.3154178,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '224.0.0.251': {'last_updated': 1708862316.346095,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.234': {'last_updated': 1708862316.3698118,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '255.255.255.255': {'last_updated': 1708862316.3873026,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.103': {'last_updated': 1708862316.4001026,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '34.140.193.14': {'last_updated': 1708862316.4166403,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.1': {'last_updated': 1708862316.4345937,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '52.58.249.45': {'last_updated': 1708862316.4526014,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.219': {'last_updated': 1708862316.4854052,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '18.157.203.183': {'last_updated': 1708862316.5129883,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.222': {'last_updated': 1708862316.545707,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '35.158.31.135': {'last_updated': 1708862316.572727,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '99.84.146.103': {'last_updated': 1708862316.585825,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '0.0.0.0': {'last_updated': 1708862316.5971653,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.146': {'last_updated': 1708862316.6090422,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '54.157.113.19': {'last_updated': 1708862316.6246989,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '99.84.146.36': {'last_updated': 1708862316.6425116,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '144.76.59.37': {'last_updated': 1708862316.6674533,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.179': {'last_updated': 1708862316.6969705,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '52.215.134.83': {'last_updated': 1708862316.7248495,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.218': {'last_updated': 1708862316.7446759,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '99.84.146.12': {'last_updated': 1708862316.7668028,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '35.173.146.140': {'last_updated': 1708862316.7786818,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.53.103.108': {'last_updated': 1708862316.7897837,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '168.119.4.163': {'last_updated': 1708862316.8010297,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '185.248.189.10': {'last_updated': 1708862316.8133514,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.203': {'last_updated': 1708862316.8331127,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '99.84.146.13': {'last_updated': 1708862316.8717947,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '34.239.12.200': {'last_updated': 1708862316.9061527,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.212': {'last_updated': 1708862316.933636,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '224.0.1.187': {'last_updated': 1708862316.9574904,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '224.0.0.1': {'last_updated': 1708862316.9734826,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '216.239.35.12': {'last_updated': 1708862316.991065,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '18.184.180.174': {'last_updated': 1708862317.0031495,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '172.104.134.72': {'last_updated': 1708862317.0218375,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '3.86.119.99': {'last_updated': 1708862317.0376055,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '192.168.1.237': {'last_updated': 1708862317.0575688,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '52.57.38.165': {'last_updated': 1708862317.085335,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '18.195.249.137': {'last_updated': 1708862317.115422,\n", " 'results': {'incoming_attack_probability': 0}},\n", " '52.205.6.197': {'last_updated': 1708862317.1521063,\n", " 'results': {'incoming_attack_probability': 0}}}" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 7 } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.6" } }, "nbformat": 4, "nbformat_minor": 5 }