initial commit
This commit is contained in:
@@ -0,0 +1,572 @@
|
||||
### JetBrains template
|
||||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
|
||||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
||||
|
||||
# User-specific stuff
|
||||
.idea/**/workspace.xml
|
||||
.idea/**/tasks.xml
|
||||
.idea/**/usage.statistics.xml
|
||||
.idea/**/dictionaries
|
||||
.idea/**/shelf
|
||||
|
||||
# AWS User-specific
|
||||
.idea/**/aws.xml
|
||||
|
||||
# Generated files
|
||||
.idea/**/contentModel.xml
|
||||
|
||||
# Sensitive or high-churn files
|
||||
.idea/**/dataSources/
|
||||
.idea/**/dataSources.ids
|
||||
.idea/**/dataSources.local.xml
|
||||
.idea/**/sqlDataSources.xml
|
||||
.idea/**/dynamic.xml
|
||||
.idea/**/uiDesigner.xml
|
||||
.idea/**/dbnavigator.xml
|
||||
|
||||
# Gradle
|
||||
.idea/**/gradle.xml
|
||||
.idea/**/libraries
|
||||
|
||||
# Gradle and Maven with auto-import
|
||||
# When using Gradle or Maven with auto-import, you should exclude module files,
|
||||
# since they will be recreated, and may cause churn. Uncomment if using
|
||||
# auto-import.
|
||||
# .idea/artifacts
|
||||
# .idea/compiler.xml
|
||||
# .idea/jarRepositories.xml
|
||||
# .idea/modules.xml
|
||||
# .idea/*.iml
|
||||
# .idea/modules
|
||||
# *.iml
|
||||
# *.ipr
|
||||
|
||||
# CMake
|
||||
cmake-build-*/
|
||||
|
||||
# Mongo Explorer plugin
|
||||
.idea/**/mongoSettings.xml
|
||||
|
||||
# File-based project format
|
||||
*.iws
|
||||
|
||||
# IntelliJ
|
||||
out/
|
||||
|
||||
# mpeltonen/sbt-idea plugin
|
||||
.idea_modules/
|
||||
|
||||
# JIRA plugin
|
||||
atlassian-ide-plugin.xml
|
||||
|
||||
# Cursive Clojure plugin
|
||||
.idea/replstate.xml
|
||||
|
||||
# SonarLint plugin
|
||||
.idea/sonarlint/
|
||||
|
||||
# Crashlytics plugin (for Android Studio and IntelliJ)
|
||||
com_crashlytics_export_strings.xml
|
||||
crashlytics.properties
|
||||
crashlytics-build.properties
|
||||
fabric.properties
|
||||
|
||||
# Editor-based Rest Client
|
||||
.idea/httpRequests
|
||||
|
||||
# Android studio 3.1+ serialized cache file
|
||||
.idea/caches/build_file_checksums.ser
|
||||
|
||||
### VirtualEnv template
|
||||
# Virtualenv
|
||||
# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
|
||||
.Python
|
||||
[Bb]in
|
||||
[Ii]nclude
|
||||
[Ll]ib
|
||||
[Ll]ib64
|
||||
[Ll]ocal
|
||||
[Ss]cripts
|
||||
pyvenv.cfg
|
||||
.venv
|
||||
pip-selfcheck.json
|
||||
|
||||
### TeX template
|
||||
## Core latex/pdflatex auxiliary files:
|
||||
*.aux
|
||||
*.lof
|
||||
*.log
|
||||
*.lot
|
||||
*.fls
|
||||
*.out
|
||||
*.toc
|
||||
*.fmt
|
||||
*.fot
|
||||
*.cb
|
||||
*.cb2
|
||||
.*.lb
|
||||
|
||||
## Intermediate documents:
|
||||
*.dvi
|
||||
*.xdv
|
||||
*-converted-to.*
|
||||
# these rules might exclude image files for figures etc.
|
||||
# *.ps
|
||||
# *.eps
|
||||
# *.pdf
|
||||
|
||||
## Generated if empty string is given at "Please type another file name for output:"
|
||||
.pdf
|
||||
|
||||
## Bibliography auxiliary files (bibtex/biblatex/biber):
|
||||
*.bbl
|
||||
*.bcf
|
||||
*.blg
|
||||
*-blx.aux
|
||||
*-blx.bib
|
||||
*.run.xml
|
||||
|
||||
## Build tool auxiliary files:
|
||||
*.fdb_latexmk
|
||||
*.synctex
|
||||
*.synctex(busy)
|
||||
*.synctex.gz
|
||||
*.synctex.gz(busy)
|
||||
*.pdfsync
|
||||
|
||||
## Build tool directories for auxiliary files
|
||||
# latexrun
|
||||
latex.out/
|
||||
|
||||
## Auxiliary and intermediate files from other packages:
|
||||
# algorithms
|
||||
*.alg
|
||||
*.loa
|
||||
|
||||
# achemso
|
||||
acs-*.bib
|
||||
|
||||
# amsthm
|
||||
*.thm
|
||||
|
||||
# beamer
|
||||
*.nav
|
||||
*.pre
|
||||
*.snm
|
||||
*.vrb
|
||||
|
||||
# changes
|
||||
*.soc
|
||||
|
||||
# comment
|
||||
*.cut
|
||||
|
||||
# cprotect
|
||||
*.cpt
|
||||
|
||||
# elsarticle (documentclass of Elsevier journals)
|
||||
*.spl
|
||||
|
||||
# endnotes
|
||||
*.ent
|
||||
|
||||
# fixme
|
||||
*.lox
|
||||
|
||||
# feynmf/feynmp
|
||||
*.mf
|
||||
*.mp
|
||||
*.t[1-9]
|
||||
*.t[1-9][0-9]
|
||||
*.tfm
|
||||
|
||||
#(r)(e)ledmac/(r)(e)ledpar
|
||||
*.end
|
||||
*.?end
|
||||
*.[1-9]
|
||||
*.[1-9][0-9]
|
||||
*.[1-9][0-9][0-9]
|
||||
*.[1-9]R
|
||||
*.[1-9][0-9]R
|
||||
*.[1-9][0-9][0-9]R
|
||||
*.eledsec[1-9]
|
||||
*.eledsec[1-9]R
|
||||
*.eledsec[1-9][0-9]
|
||||
*.eledsec[1-9][0-9]R
|
||||
*.eledsec[1-9][0-9][0-9]
|
||||
*.eledsec[1-9][0-9][0-9]R
|
||||
|
||||
# glossaries
|
||||
*.acn
|
||||
*.acr
|
||||
*.glg
|
||||
*.glo
|
||||
*.gls
|
||||
*.glsdefs
|
||||
*.lzo
|
||||
*.lzs
|
||||
*.slg
|
||||
*.slo
|
||||
*.sls
|
||||
|
||||
# uncomment this for glossaries-extra (will ignore makeindex's style files!)
|
||||
# *.ist
|
||||
|
||||
# gnuplot
|
||||
*.gnuplot
|
||||
*.table
|
||||
|
||||
# gnuplottex
|
||||
*-gnuplottex-*
|
||||
|
||||
# gregoriotex
|
||||
*.gaux
|
||||
*.glog
|
||||
*.gtex
|
||||
|
||||
# htlatex
|
||||
*.4ct
|
||||
*.4tc
|
||||
*.idv
|
||||
*.lg
|
||||
*.trc
|
||||
*.xref
|
||||
|
||||
# hyperref
|
||||
*.brf
|
||||
|
||||
# knitr
|
||||
*-concordance.tex
|
||||
# TODO Uncomment the next line if you use knitr and want to ignore its generated tikz files
|
||||
# *.tikz
|
||||
*-tikzDictionary
|
||||
|
||||
# listings
|
||||
*.lol
|
||||
|
||||
# luatexja-ruby
|
||||
*.ltjruby
|
||||
|
||||
# makeidx
|
||||
*.idx
|
||||
*.ilg
|
||||
*.ind
|
||||
|
||||
# minitoc
|
||||
*.maf
|
||||
*.mlf
|
||||
*.mlt
|
||||
*.mtc[0-9]*
|
||||
*.slf[0-9]*
|
||||
*.slt[0-9]*
|
||||
*.stc[0-9]*
|
||||
|
||||
# minted
|
||||
_minted*
|
||||
*.pyg
|
||||
|
||||
# morewrites
|
||||
*.mw
|
||||
|
||||
# newpax
|
||||
*.newpax
|
||||
|
||||
# nomencl
|
||||
*.nlg
|
||||
*.nlo
|
||||
*.nls
|
||||
|
||||
# pax
|
||||
*.pax
|
||||
|
||||
# pdfpcnotes
|
||||
*.pdfpc
|
||||
|
||||
# sagetex
|
||||
*.sagetex.sage
|
||||
*.sagetex.py
|
||||
*.sagetex.scmd
|
||||
|
||||
# scrwfile
|
||||
*.wrt
|
||||
|
||||
# svg
|
||||
svg-inkscape/
|
||||
|
||||
# sympy
|
||||
*.sout
|
||||
*.sympy
|
||||
sympy-plots-for-*.tex/
|
||||
|
||||
# pdfcomment
|
||||
*.upa
|
||||
*.upb
|
||||
|
||||
# pythontex
|
||||
*.pytxcode
|
||||
pythontex-files-*/
|
||||
|
||||
# tcolorbox
|
||||
*.listing
|
||||
|
||||
# thmtools
|
||||
*.loe
|
||||
|
||||
# TikZ & PGF
|
||||
*.dpth
|
||||
*.md5
|
||||
*.auxlock
|
||||
|
||||
# titletoc
|
||||
*.ptc
|
||||
|
||||
# todonotes
|
||||
*.tdo
|
||||
|
||||
# vhistory
|
||||
*.hst
|
||||
*.ver
|
||||
|
||||
# easy-todo
|
||||
*.lod
|
||||
|
||||
# xcolor
|
||||
*.xcp
|
||||
|
||||
# xmpincl
|
||||
*.xmpi
|
||||
|
||||
# xindy
|
||||
*.xdy
|
||||
|
||||
# xypic precompiled matrices and outlines
|
||||
*.xyc
|
||||
*.xyd
|
||||
|
||||
# endfloat
|
||||
*.ttt
|
||||
*.fff
|
||||
|
||||
# Latexian
|
||||
TSWLatexianTemp*
|
||||
|
||||
## Editors:
|
||||
# WinEdt
|
||||
*.bak
|
||||
*.sav
|
||||
|
||||
# Texpad
|
||||
.texpadtmp
|
||||
|
||||
# LyX
|
||||
*.lyx~
|
||||
|
||||
# Kile
|
||||
*.backup
|
||||
|
||||
# gummi
|
||||
.*.swp
|
||||
|
||||
# KBibTeX
|
||||
*~[0-9]*
|
||||
|
||||
# TeXnicCenter
|
||||
*.tps
|
||||
|
||||
# auto folder when using emacs and auctex
|
||||
./auto/*
|
||||
*.el
|
||||
|
||||
# expex forward references with \gathertags
|
||||
*-tags.tex
|
||||
|
||||
# standalone packages
|
||||
*.sta
|
||||
|
||||
# Makeindex log files
|
||||
*.lpz
|
||||
|
||||
# xwatermark package
|
||||
*.xwm
|
||||
|
||||
# REVTeX puts footnotes in the bibliography by default, unless the nofootinbib
|
||||
# option is specified. Footnotes are the stored in a file with suffix Notes.bib.
|
||||
# Uncomment the next line to have this generated file ignored.
|
||||
#*Notes.bib
|
||||
|
||||
### JupyterNotebooks template
|
||||
# gitignore template for Jupyter Notebooks
|
||||
# website: http://jupyter.org/
|
||||
|
||||
.ipynb_checkpoints
|
||||
*/.ipynb_checkpoints/*
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# Remove previous ipynb_checkpoints
|
||||
# git rm -r .ipynb_checkpoints/
|
||||
|
||||
### Python template
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/#use-with-ide
|
||||
.pdm.toml
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
version: '3'
|
||||
services:
|
||||
zookeeper:
|
||||
image: confluentinc/cp-zookeeper:latest
|
||||
container_name: zookeeper
|
||||
ports:
|
||||
- "2181:2181"
|
||||
environment:
|
||||
ZOOKEEPER_CLIENT_PORT: 2181
|
||||
ZOOKEEPER_TICK_TIME: 2000
|
||||
|
||||
kafka:
|
||||
image: confluentinc/cp-kafka:latest
|
||||
container_name: kafka
|
||||
depends_on:
|
||||
- zookeeper
|
||||
ports:
|
||||
- "9092:9092"
|
||||
environment:
|
||||
KAFKA_BROKER_ID: 1
|
||||
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
||||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
||||
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
|
||||
AUTO_CREATE_TOPICS: true
|
||||
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
||||
|
||||
|
||||
redis:
|
||||
image: redis:latest
|
||||
container_name: redis
|
||||
ports:
|
||||
- "6379:6379"
|
||||
@@ -0,0 +1,158 @@
|
||||
2024-02-27 13:51:08,214 - Packet-Worker 7ff4aeaa-44e2-4a1a-a9bd-f1eda3ea8f7d - INFO - Worker started
|
||||
2024-02-27 13:51:08,422 - Packet-Worker 7ff4aeaa-44e2-4a1a-a9bd-f1eda3ea8f7d - INFO - Initialized, waiting for packets
|
||||
2024-02-27 13:56:03,544 - Packet-Worker 25d1f9c9-b5ae-4e3d-b5d4-0d54376386cf - INFO - Worker started
|
||||
2024-02-27 13:56:03,748 - Packet-Worker 25d1f9c9-b5ae-4e3d-b5d4-0d54376386cf - INFO - Initialized, waiting for packets
|
||||
2024-02-27 13:56:22,895 - State-Merger 24d87082-2bef-415a-8e16-3b2f80dcd694 - INFO - Worker started
|
||||
2024-02-27 13:56:22,998 - State-Merger 24d87082-2bef-415a-8e16-3b2f80dcd694 - INFO - Connected to kafka
|
||||
2024-02-27 13:56:22,998 - State-Merger 24d87082-2bef-415a-8e16-3b2f80dcd694 - INFO - Connected to redis
|
||||
2024-02-27 13:56:22,998 - State-Merger 24d87082-2bef-415a-8e16-3b2f80dcd694 - INFO - Initialized, waiting for states
|
||||
2024-02-27 13:56:34,021 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - Connected to kafka
|
||||
2024-02-27 13:56:34,021 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - Connected to redis
|
||||
2024-02-27 13:56:34,021 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - Initialized, waiting for states
|
||||
2024-02-27 13:56:34,029 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:35,030 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:36,030 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:37,030 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:38,031 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:39,031 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:40,032 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:41,032 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:42,032 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:42,579 - Connection-Analysis f714e50d-189a-443b-a668-ecd30dd35d71 - INFO - Connection-Analysis f714e50d-189a-443b-a668-ecd30dd35d71 started
|
||||
2024-02-27 13:56:42,579 - Connection-Analysis f714e50d-189a-443b-a668-ecd30dd35d71 - INFO - Loading analysis models
|
||||
2024-02-27 13:56:42,958 - Connection-Analysis f714e50d-189a-443b-a668-ecd30dd35d71 - INFO - Loaded model incoming_attack_probability
|
||||
2024-02-27 13:56:43,033 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:44,033 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:44,620 - Connection-Analysis f714e50d-189a-443b-a668-ecd30dd35d71 - INFO - Loaded model outgoing_attack_probability
|
||||
2024-02-27 13:56:45,033 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:46,033 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:47,034 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:48,034 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:49,034 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:50,035 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:51,035 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:52,035 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:53,036 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:54,036 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:55,036 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:56,036 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:57,037 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:58,037 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:56:59,037 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:00,038 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:01,038 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:01,460 - Packet-Worker d750d3d9-8562-413f-a53c-7a0a869a56bb - INFO - Worker started
|
||||
2024-02-27 13:57:01,664 - Packet-Worker d750d3d9-8562-413f-a53c-7a0a869a56bb - INFO - Initialized, waiting for packets
|
||||
2024-02-27 13:57:02,038 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:03,039 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:04,039 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:05,039 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:05,671 - State-Merger db6e42e9-1c66-4d64-ba60-a4be8bb3cbd1 - INFO - Worker started
|
||||
2024-02-27 13:57:05,774 - State-Merger db6e42e9-1c66-4d64-ba60-a4be8bb3cbd1 - INFO - Connected to kafka
|
||||
2024-02-27 13:57:05,774 - State-Merger db6e42e9-1c66-4d64-ba60-a4be8bb3cbd1 - INFO - Connected to redis
|
||||
2024-02-27 13:57:05,775 - State-Merger db6e42e9-1c66-4d64-ba60-a4be8bb3cbd1 - INFO - Initialized, waiting for states
|
||||
2024-02-27 13:57:06,040 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:07,040 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:08,040 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:09,041 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:10,041 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:11,041 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:12,042 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:13,042 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:14,043 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:15,043 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:16,043 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:17,044 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:18,044 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:19,044 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:20,045 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:21,045 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:22,046 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:23,046 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:24,046 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:25,047 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:26,047 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:27,047 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:28,047 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:29,048 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:30,048 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:31,048 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:32,049 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:33,049 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:34,049 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:35,049 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:36,050 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:37,050 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:38,050 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:39,051 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:40,051 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:41,051 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:42,052 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:43,052 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:44,052 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:45,052 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:46,053 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:47,053 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:48,053 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:48,206 - Connection-Analysis eac8a4c6-d064-40fc-95d2-9b0828454286 - INFO - Connection-Analysis eac8a4c6-d064-40fc-95d2-9b0828454286 started
|
||||
2024-02-27 13:57:48,206 - Connection-Analysis eac8a4c6-d064-40fc-95d2-9b0828454286 - INFO - Loading analysis models
|
||||
2024-02-27 13:57:48,397 - Connection-Analysis eac8a4c6-d064-40fc-95d2-9b0828454286 - INFO - Loaded model incoming_attack_probability
|
||||
2024-02-27 13:57:49,054 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:49,248 - Connection-Analysis eac8a4c6-d064-40fc-95d2-9b0828454286 - INFO - Loaded model outgoing_attack_probability
|
||||
2024-02-27 13:57:50,054 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:51,054 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:52,055 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:53,055 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:54,055 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:55,056 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:56,056 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:57,057 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:58,057 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:57:59,057 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:00,058 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:01,058 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:02,058 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:03,058 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:04,059 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:05,059 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:06,059 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:07,060 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:08,060 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:09,060 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:10,061 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:11,061 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:12,061 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:13,062 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:14,062 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:15,062 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:16,063 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:17,063 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:18,063 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:19,064 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:20,064 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:21,064 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:22,065 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:23,065 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:24,066 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:25,066 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:26,066 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:27,067 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:28,067 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:29,067 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:30,068 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:31,068 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:32,068 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:33,069 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:34,069 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:35,070 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:36,070 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - No network state found, waiting for state
|
||||
2024-02-27 13:58:36,544 - Packet-Worker d750d3d9-8562-413f-a53c-7a0a869a56bb - INFO - Processing 581 packets
|
||||
2024-02-27 13:58:36,544 - Packet-Worker d750d3d9-8562-413f-a53c-7a0a869a56bb - INFO - Processing 581 packets in 1 processing steps
|
||||
2024-02-27 13:58:36,559 - Packet-Worker d750d3d9-8562-413f-a53c-7a0a869a56bb - INFO - Processed 581 packets
|
||||
2024-02-27 13:58:36,666 - State-Merger db6e42e9-1c66-4d64-ba60-a4be8bb3cbd1 - INFO - Processing 1 state(s)
|
||||
2024-02-27 13:58:36,667 - State-Merger db6e42e9-1c66-4d64-ba60-a4be8bb3cbd1 - INFO - Merged 1 state(s), waiting for next batch
|
||||
2024-02-27 13:58:37,070 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - Found 23 nodes in network state
|
||||
2024-02-27 13:58:37,171 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - Found 23 new nodes in network state
|
||||
2024-02-27 13:58:37,224 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - Dispatched 23 new node updates
|
||||
2024-02-27 13:58:37,347 - Connection-Analysis eac8a4c6-d064-40fc-95d2-9b0828454286 - INFO - Analyzing 23 nodes
|
||||
2024-02-27 13:58:37,428 - Work-Dispatcher b7dd6b52-71aa-4f58-b292-d44f8b9cd9c1 - INFO - Found 23 nodes with awaiting results
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "37ac6be2-a15f-40db-ba98-7f4559cf5749",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df = pd.read_pickle('../echo-dot-activity.pickel')\n",
|
||||
"\n",
|
||||
"print(\"ip/ip.dst\" in df.columns)\n",
|
||||
"\n",
|
||||
"df['Activity'] = pd.Series(False, index=df.index)\n",
|
||||
"\n",
|
||||
"df.loc[2330 - 12198, 'Activity'] = True # music listening\n",
|
||||
"df.loc[12744 - 13396, 'Activity'] = True # announcement\n",
|
||||
"df.loc[13548 - 20533, 'Activity'] = True # key exchange? maybe not activity...\n",
|
||||
"df.loc[20556 - 26757, 'Activity'] = True # music listening\n",
|
||||
"df.loc[26813 - 26825, 'Activity'] = True # announcement\n",
|
||||
"df.loc[26918 - 26936, 'Activity'] = True # volume adjustement \n",
|
||||
"\n",
|
||||
"df.to_pickle('../echo-dot-activity.pickel')\n"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"id": "fe1e40dd56e78b5"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df = pd.read_pickle(\"../echo-dot-attack.pickel\")\n",
|
||||
"\n",
|
||||
"df[\"Attack\"] = pd.Series(False, index=df.index, dtype=int)\n",
|
||||
"\n",
|
||||
"print(\"ip/ip.dst\" in df.columns)\n",
|
||||
"\n",
|
||||
"df.loc[49 - 262255, \"Attack\"] = 2\n",
|
||||
"df.loc[262258 - 622810, \"Attack\"] = 2\n",
|
||||
"df.loc[624338 - 624363, \"Attack\"] = 1\n",
|
||||
"df.loc[624541 - 627011, \"Attack\"] = 3\n",
|
||||
"\n",
|
||||
"df.to_pickle(\"../echo-dot-attack.pickel\")"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"id": "d6a43f9140b4e1f9"
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.9"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "initial_id",
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df = pd.read_pickle(\"../echo-dot-attack.pickel\")"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"id": "36cbb373e91e7b86"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# get unique src ip addresses, drop nans\n",
|
||||
"ip_addresses = df[\"ip/ip.src\"].unique()\n",
|
||||
"ip_addresses = ip_addresses[~pd.isnull(ip_addresses)]\n",
|
||||
"ip_addresses"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"id": "737d862bee140f41"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# get associated mac addresses\n",
|
||||
"associated_mac_addresses = list(set(\n",
|
||||
" [df[df[\"ip/ip.src\"] == ip][\"eth/eth.src\"].unique()[0] for ip in ip_addresses]))\n",
|
||||
"associated_mac_addresses"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"id": "f5f01e284c2fd677"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# get unique mac addresses, drop nans\n",
|
||||
"mac_addresses = set(df[\"eth/eth.src\"].unique()).union(set(df[\"eth/eth.dst\"].unique()))\n",
|
||||
"mac_addresses = [mac for mac in mac_addresses if not pd.isnull(mac)]\n",
|
||||
"mac_addresses"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"id": "ba1b9681e60d31b7"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# check if all mac addresses are associated with an ip address\n",
|
||||
"for mac in mac_addresses:\n",
|
||||
" if mac not in associated_mac_addresses:\n",
|
||||
" print(f\"mac address {mac} is not associated with an ip address\")"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"id": "f6251b2b6b4f71cf"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# create new columns in dataframe called ip/ip.dst with ip address associated to destination mac"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"id": "1c5d264aff0955cc"
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 2
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython2",
|
||||
"version": "2.7.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,377 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "55bc224f879f844a",
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import xml.etree.ElementTree\n",
|
||||
"import xml.etree.ElementTree as ET\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"DEVICES_LAB = {\n",
|
||||
" # 'a8:03:2a:b1:35:60': 'dev01-shelly-plus-1pm-relais', # Christoph\n",
|
||||
" # '60:01:94:c7:69:ac': 'dev02-sonoff-relais', # Oliver\n",
|
||||
" # 'd8:f1:5b:d8:08:0c': 'dev03-tuya-lampe', # Leonard\n",
|
||||
" # '1c:d6:bd:b5:d3:bb': 'dev04-linkind-zigbee-mini-hub', # Thorsten\n",
|
||||
" # '6c:5a:b0:7d:e2:25': 'dev05-tplink-tapo-l530e-birne', # Suad\n",
|
||||
" # '54:af:97:7c:5e:f0': 'dev06-tapo-steckdose', # Andreas\n",
|
||||
" # '74:ab:93:de:a0:7e': 'dev07-blink-sicherheitskamera', # Julian\n",
|
||||
" '34:25:be:ef:91:bf': 'dev08-echo-dot-l4s3re', # Alexander\n",
|
||||
" # '70:ee:50:90:64:04': 'dev09-netatmo-smart-weather-station', # Jakobus\n",
|
||||
" # '68:3a:48:4b:53:c5': 'dev10-aeotec-z-wave-hub', # Andre\n",
|
||||
" # 'dc:ed:83:4a:cf:76': 'dev11-aqara-presence-sensor-fp2', # Artur\n",
|
||||
" # '8c:f6:81:dc:63:54': 'dev12-shelly-bewegungsmelder', # Moritz\n",
|
||||
" # '24:4c:ab:43:0d:0f': 'dev13-shelly-flood', # Mohamad\n",
|
||||
" # '08:b6:1f:cc:4d:c0': 'dev14-shelly-ht-temperatur-sensor', # Bastian\n",
|
||||
" # '90:48:6c:17:ae:25': 'dev15-ring-door-camera' # Victor (Ring hinzugefügt, device Nummern angepasst)\n",
|
||||
"}"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"id": "b5715a859aec218c"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "39ad609a78e03fa0",
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"MINIMAL_FEATURES = [\n",
|
||||
" #Meta\n",
|
||||
" '_ws.malformed/_ws.expert/_ws.malformed.expert',\n",
|
||||
" # 'bootp/bootp.dhcp'\n",
|
||||
" # 'bootp/bootp.flags.bc'\n",
|
||||
" # 'bootp/bootp.flags.reserved'\n",
|
||||
" # 'bootp/bootp.hops'\n",
|
||||
" # 'bootp/bootp.hw.len'\n",
|
||||
" # 'bootp/bootp.hw.type'\n",
|
||||
" # 'bootp/bootp.secs'\n",
|
||||
" # 'bootp/bootp.type'\n",
|
||||
" 'classicstun.length',\n",
|
||||
" 'classicstun.type',\n",
|
||||
" # 'frame/frame.cap_len'\n",
|
||||
" 'frame/frame.number', # Nur für Labeling\n",
|
||||
" 'frame/frame.encap_type',\n",
|
||||
" 'frame/frame.ignore',\n",
|
||||
" 'frame/frame.len',\n",
|
||||
" 'frame/frame.marked',\n",
|
||||
" 'frame/frame.offset_shift',\n",
|
||||
" 'frame/frame.packet_flags',\n",
|
||||
" # 'geninfo/caplen'\n",
|
||||
" # 'geninfo/len'\n",
|
||||
" 'geninfo/timestamp',\n",
|
||||
" #Data-Link\n",
|
||||
" 'eth/eth.dst',\n",
|
||||
" 'eth/eth.dst/eth.dst.ig',\n",
|
||||
" 'eth/eth.dst/eth.dst.oui',\n",
|
||||
" 'eth/eth.dst/eth.lg',\n",
|
||||
" 'eth/eth.src',\n",
|
||||
" 'eth/eth.src/eth.ig',\n",
|
||||
" 'eth/eth.src/eth.lg',\n",
|
||||
" 'eth/eth.src/eth.src.oui',\n",
|
||||
" 'eth/eth.type',\n",
|
||||
" 'arp/arp.hw.type', # Auffälliges Verhalten des Echo (5 Felder aus Paper zurück)\n",
|
||||
" 'arp/arp.proto.type',\n",
|
||||
" 'arp/arp.hw.size',\n",
|
||||
" 'arp/arp.proto.size',\n",
|
||||
" 'arp/arp.opcode',\n",
|
||||
" #Network\n",
|
||||
" 'icmp/data/data.len',\n",
|
||||
" 'icmp/icmp.checksum.status',\n",
|
||||
" 'icmp/icmp.code',\n",
|
||||
" 'icmp/icmp.ident',\n",
|
||||
" 'icmp/icmp.resp_in',\n",
|
||||
" 'icmp/icmp.resp_to',\n",
|
||||
" 'icmp/icmp.seq',\n",
|
||||
" 'icmp/icmp.seq_le',\n",
|
||||
" 'icmp/icmp.type',\n",
|
||||
" 'icmp/icmp.udp/icmp.udp.dstport',\n",
|
||||
" 'icmp/icmp.udp/icmp.udp.length',\n",
|
||||
" 'icmp/icmp.udp/icmp.udp.srcport',\n",
|
||||
" 'igmp/igmp.checksum.status',\n",
|
||||
" 'igmp/igmp.max_resp',\n",
|
||||
" 'igmp/igmp.maddr',\n",
|
||||
" 'igmp/igmp.type',\n",
|
||||
" 'ip/<>/ip.options.routeralert/ip.opt.ra',\n",
|
||||
" 'ip/<>/ip.options.routeralert/ip.opt.sec_cl',\n",
|
||||
" 'ip/<>/ip.options.routeralert/ip.opt.type',\n",
|
||||
" 'ip/ip.checksum.status',\n",
|
||||
" 'ip/ip.dsfield',\n",
|
||||
" 'ip/ip.dsfield/ip.dsfield.dscp',\n",
|
||||
" 'ip/ip.dsfield/ip.dsfield.ecn',\n",
|
||||
" 'ip/ip.evil_packet',\n",
|
||||
" 'ip/ip.flags',\n",
|
||||
" 'ip/ip.flags/ip.flags.df',\n",
|
||||
" 'ip/ip.flags/ip.flags.mf',\n",
|
||||
" 'ip/ip.flags/ip.flags.rb',\n",
|
||||
" 'ip/ip.frag_offset',\n",
|
||||
" 'ip/ip.hdr_len',\n",
|
||||
" 'ip/ip.id',\n",
|
||||
" 'ip/ip.len',\n",
|
||||
" 'ip/ip.proto',\n",
|
||||
" 'ip/ip.src',\n",
|
||||
" 'ip/ip.dst',\n",
|
||||
" 'ip/ip.ttl',\n",
|
||||
" 'ip/ip.version',\n",
|
||||
" # Transport\n",
|
||||
" 'tcp/tcp.ack',\n",
|
||||
" 'tcp/tcp.analysis/tcp.analysis.bytes_in_flight',\n",
|
||||
" 'tcp/tcp.analysis/tcp.analysis.push_bytes_sent',\n",
|
||||
" 'tcp/tcp.checksum.status',\n",
|
||||
" 'tcp/tcp.completeness',\n",
|
||||
" 'tcp/tcp.dstport',\n",
|
||||
" 'tcp/tcp.flags/tcp.flags.ack',\n",
|
||||
" 'tcp/tcp.flags/tcp.flags.cwr',\n",
|
||||
" 'tcp/tcp.flags/tcp.flags.ecn',\n",
|
||||
" 'tcp/tcp.flags/tcp.flags.fin',\n",
|
||||
" 'tcp/tcp.flags/tcp.flags.ns',\n",
|
||||
" 'tcp/tcp.flags/tcp.flags.push',\n",
|
||||
" 'tcp/tcp.flags/tcp.flags.res',\n",
|
||||
" 'tcp/tcp.flags/tcp.flags.reset',\n",
|
||||
" 'tcp/tcp.flags/tcp.flags.syn',\n",
|
||||
" 'tcp/tcp.flags/tcp.flags.urg',\n",
|
||||
" 'tcp/tcp.hdr_len',\n",
|
||||
" 'tcp/tcp.len',\n",
|
||||
" 'tcp/tcp.nxtseq',\n",
|
||||
" 'tcp/tcp.options/tcp.options.mss/tcp.options.mss_val',\n",
|
||||
" 'tcp/tcp.options/tcp.options.nop',\n",
|
||||
" # 'tcp/tcp.payload' \n",
|
||||
" 'tcp/tcp.seq',\n",
|
||||
" 'tcp/tcp.srcport',\n",
|
||||
" 'tcp/tcp.stream',\n",
|
||||
" 'tcp/tcp.urgent_pointer',\n",
|
||||
" 'tcp/tcp.window_size',\n",
|
||||
" 'tcp/tcp.window_size_scalefactor',\n",
|
||||
" 'tcp/tcp.window_size_value',\n",
|
||||
" 'udp/udp.checksum.status',\n",
|
||||
" 'udp/udp.dstport',\n",
|
||||
" 'udp/udp.length',\n",
|
||||
" 'udp/udp.srcport',\n",
|
||||
" 'udp/udp.stream',\n",
|
||||
"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "84b478867fed089b",
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"### PDML #######################################################\n",
|
||||
"PDML_FILE = '../dumps/usage_dump.pdml'\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"################################################################\n",
|
||||
"\n",
|
||||
"def collect_packet_fields(element: xml.etree.ElementTree.Element, prefix: str, packet_fields_: dict) -> dict:\n",
|
||||
" tag_name = element.tag\n",
|
||||
" name_attr = element.attrib['name']\n",
|
||||
" empty_name_attr = False\n",
|
||||
" if not name_attr:\n",
|
||||
" empty_name_attr = True\n",
|
||||
" col_name_ = f'{prefix}/<>' if prefix else '<>'\n",
|
||||
" else:\n",
|
||||
" col_name_ = f'{prefix}/{name_attr}' if prefix else f'{name_attr}'\n",
|
||||
"\n",
|
||||
" # if tag_name != 'proto' and col_name_ not in BLACKLISTED_FIELDS:\n",
|
||||
" if tag_name != 'proto':\n",
|
||||
" if not empty_name_attr:\n",
|
||||
" total_col_names.add(col_name_)\n",
|
||||
" try:\n",
|
||||
" packet_fields_[col_name_] = element.attrib['show']\n",
|
||||
" except KeyError:\n",
|
||||
" try:\n",
|
||||
" packet_fields_[col_name_] = element.attrib['value']\n",
|
||||
" except KeyError:\n",
|
||||
" packet_fields_[col_name_] = np.nan\n",
|
||||
"\n",
|
||||
" # Recursively process child elements (subfields)\n",
|
||||
" for child in element:\n",
|
||||
" packet_fields_.update(collect_packet_fields(child, col_name_, packet_fields_))\n",
|
||||
"\n",
|
||||
" return packet_fields_\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"print('Parsing PDML file...')\n",
|
||||
"tree = ET.parse(PDML_FILE)\n",
|
||||
"root = tree.getroot()\n",
|
||||
"\n",
|
||||
"total_col_names = set()\n",
|
||||
"total_pkt_fields = []\n",
|
||||
"\n",
|
||||
"print('Collecting packet fields...')\n",
|
||||
"# Loop over all packets\n",
|
||||
"packets = root.findall('packet')\n",
|
||||
"for packet in packets:\n",
|
||||
" packet_fields = {}\n",
|
||||
" # Loop over all protocols\n",
|
||||
" protocols = packet.findall('proto')\n",
|
||||
" for protocol in protocols:\n",
|
||||
" packet_fields.update(collect_packet_fields(protocol, '', {}))\n",
|
||||
" total_pkt_fields.append(packet_fields)\n",
|
||||
"\n",
|
||||
"total_col_names = sorted(total_col_names)\n",
|
||||
"total_pkt_fields = sorted(total_pkt_fields, key=lambda x: int(x['frame/frame.number'])) # Sort by frame.number\n",
|
||||
"df_data_full = pd.DataFrame(total_pkt_fields)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b2692056de3e85b7",
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df_data_full.shape"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "263145b13cd0b10b",
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# All columns with only NaN rows\n",
|
||||
"sorted(df_data_full.columns[df_data_full.isna().all()].tolist())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "25481a29eba04c14",
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# merged_features = set(FEATURES_FROM_PAPER + CUSTOM_CHOSEN_FEATURES)\n",
|
||||
"features = set(MINIMAL_FEATURES)\n",
|
||||
"f_intersection = features & set(df_data_full.columns)\n",
|
||||
"features_not_present_in_pdml = features - f_intersection\n",
|
||||
"# features_not_present_in_pdml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "42c984d759e144b",
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# First add all features which are present in the .pdml\n",
|
||||
"df_data_final = df_data_full.loc[:, list(f_intersection)]\n",
|
||||
"# Then add all features which have been picked and are from the paper, but are not in the .pdml. Fill them with NaNs.\n",
|
||||
"df_data_final[list(features_not_present_in_pdml)] = np.nan\n",
|
||||
"# Sort columns\n",
|
||||
"# df_data_final.sort_index(axis=1, inplace=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3454b5ca3e48ad92",
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# df_data_final.shape"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5b5914a527e2a483",
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# df_data_final.columns"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c6371cc759696ec3",
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# All columns with only NaN rows\n",
|
||||
"# df_data_final.columns[df_data_final.isna().all()].tolist()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7114a6cf74cafef8",
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# df_data_final.to_csv(\"data_final.csv\", index=False, sep='|')\n",
|
||||
"df_data_final.to_pickle(\"../echo-dot-activity.pickel\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"id": "93deee78d784393a"
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.9"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2024-02-24T13:28:00.794477Z",
|
||||
"start_time": "2024-02-24T13:28:00.782180Z"
|
||||
}
|
||||
},
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"import os\n",
|
||||
"from typing import List, Callable\n",
|
||||
"import queue\n",
|
||||
"import joblib\n",
|
||||
"import threading\n",
|
||||
"import time\n",
|
||||
"import logging\n",
|
||||
"import json\n",
|
||||
"\n",
|
||||
"import networkx as nx\n",
|
||||
"\n",
|
||||
"from kafka import KafkaConsumer, KafkaProducer\n",
|
||||
"import redis\n",
|
||||
"\n",
|
||||
"from graph_based_intrusion_detection.utils import constants\n",
|
||||
"from graph_based_intrusion_detection.utils.kafka_utils import deserialize, serialize\n",
|
||||
"from graph_based_intrusion_detection.utils.logging import create_logger\n",
|
||||
"from graph_based_intrusion_detection.utils.state_merging import merge_states\n",
|
||||
"from graph_based_intrusion_detection.packet_processing import processing_functions\n",
|
||||
"from graph_based_intrusion_detection.packet_processing.processing_functions import process_packets\n",
|
||||
"from graph_based_intrusion_detection.graph_processing.data_extraction import create_dataset_for_node"
|
||||
],
|
||||
"id": "48d278a5f5448c0e",
|
||||
"outputs": [],
|
||||
"execution_count": 16
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2024-02-23T14:28:05.510736Z",
|
||||
"start_time": "2024-02-23T14:28:05.507164Z"
|
||||
}
|
||||
},
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# define constants\n",
|
||||
"log_file_path = os.path.abspath(os.path.join(os.path.dirname(\".\"), \"logs\", \"main.log\"))\n",
|
||||
"print(f\"Log file path: {log_file_path}\")\n",
|
||||
"\n",
|
||||
"update_interval_in_ms = 1000\n",
|
||||
"\n"
|
||||
],
|
||||
"id": "bbed502fad42ce8f",
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Log file path: /home/alex/projects/it-security-praktikum/devices/dev08-echo-dot-l4s3re/graph_based_intrusion_detection/notebooks/logs/main.log\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"execution_count": 11
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2024-02-23T14:13:07.915623Z",
|
||||
"start_time": "2024-02-23T14:13:07.912275Z"
|
||||
}
|
||||
},
|
||||
"cell_type": "code",
|
||||
"source": "# define function for creating logger for each thread\n",
|
||||
"id": "44cf10deef903383",
|
||||
"outputs": [],
|
||||
"execution_count": 8
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7759e6a6c4b91cd6",
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"packet_queue = queue.Queue()\n",
|
||||
"state_queue = queue.Queue()\n",
|
||||
"connection_update_queue = queue.Queue()\n",
|
||||
"connection_update_done_queue = queue.Queue()\n",
|
||||
"\n",
|
||||
"current_state = dict()\n",
|
||||
"state_lock = threading.Lock()\n",
|
||||
"\n",
|
||||
"connection_registry = dict()\n",
|
||||
"connection_registry_lock = threading.Lock()\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metadata": {},
|
||||
"cell_type": "code",
|
||||
"outputs": [],
|
||||
"execution_count": null,
|
||||
"source": [
|
||||
"def packet_worker(packet_processing_functions: list, worker_id: int):\n",
|
||||
" \"\"\"\n",
|
||||
" Worker thread, processes packets from the packet queue and puts the resulting state into the state queue\n",
|
||||
" :param packet_processing_functions: function for processing packets\n",
|
||||
" :param worker_id: id of the worker\n",
|
||||
" :return: None\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" logger = create_logger(f\"Packet-Worker {worker_id}\")\n",
|
||||
" logger.info(f\"Worker {worker_id} started\")\n",
|
||||
"\n",
|
||||
" while True:\n",
|
||||
" # get current batch size based on approximate queue length\n",
|
||||
" n_packets = packet_queue.qsize()\n",
|
||||
" batch_size = n_packets // n_workers\n",
|
||||
" logger.info(f\"Worker {worker_id} current batch size: {batch_size}\")\n",
|
||||
"\n",
|
||||
" # fetch batch from queue\n",
|
||||
" batch = list()\n",
|
||||
" try:\n",
|
||||
" for _ in range(batch_size):\n",
|
||||
" batch.append(packet_queue.get_nowait())\n",
|
||||
" except queue.Empty:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
" logger.info(f\"Worker {worker_id} processing {len(batch)} packets\")\n",
|
||||
" # process batch\n",
|
||||
" if len(batch) > 0:\n",
|
||||
" state = process_packets(batch, packet_processing_functions)\n",
|
||||
" state_queue.put(state)\n",
|
||||
" else:\n",
|
||||
" time.sleep(0.1)\n",
|
||||
"\n",
|
||||
" logger.info(f\"Worker {worker_id} done\")\n",
|
||||
"\n",
|
||||
" # mark batch as done\n",
|
||||
" for _ in batch:\n",
|
||||
" packet_queue.task_done()"
|
||||
],
|
||||
"id": "852ee0a61981f880"
|
||||
},
|
||||
{
|
||||
"metadata": {},
|
||||
"cell_type": "code",
|
||||
"outputs": [],
|
||||
"execution_count": null,
|
||||
"source": [
|
||||
"\n",
|
||||
"def merger(merger_function: Callable, worker_id: int):\n",
|
||||
" \"\"\"\n",
|
||||
" Merger thread, merges states from the state queue into one central state representing the state of the network\n",
|
||||
" :param merger_function: function for merging states\n",
|
||||
" :param worker_id: id of the worker\n",
|
||||
" :return: None\n",
|
||||
" \"\"\"\n",
|
||||
" global current_state\n",
|
||||
"\n",
|
||||
" logger = create_logger(f\"Merger {worker_id}\")\n",
|
||||
" logger.info(f\"Merger {worker_id} started\")\n",
|
||||
"\n",
|
||||
" while True:\n",
|
||||
" # get state lock\n",
|
||||
" state_lock.acquire()\n",
|
||||
"\n",
|
||||
" # fetch state from queue\n",
|
||||
" state = state_queue.get()\n",
|
||||
"\n",
|
||||
" logger.info(f\"Merging state\")\n",
|
||||
" states = [current_state, state]\n",
|
||||
" # merge state\n",
|
||||
" new_state = merger_function(states)\n",
|
||||
"\n",
|
||||
" # update state\n",
|
||||
" current_state = new_state\n",
|
||||
"\n",
|
||||
" # mark state as done\n",
|
||||
" state_queue.task_done()\n",
|
||||
"\n",
|
||||
" # release state lock\n",
|
||||
" state_lock.release()\n",
|
||||
"\n",
|
||||
" logger.info(f\"State merged\")\n"
|
||||
],
|
||||
"id": "initial_id"
|
||||
},
|
||||
{
|
||||
"metadata": {},
|
||||
"cell_type": "code",
|
||||
"outputs": [],
|
||||
"execution_count": null,
|
||||
"source": [
|
||||
"def dispatcher(packet: dict,\n",
|
||||
" worker_id: int):\n",
|
||||
" \"\"\"\n",
|
||||
" Dispatcher thread, watches connection registry and dispatches connection to workers for analysis\n",
|
||||
" :param packet: packet to dispatch\n",
|
||||
" :param worker_id: id of the worker\n",
|
||||
" :return: None\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" logger = create_logger(f\"Dispatcher {worker_id}\")\n",
|
||||
" logger.info(f\"Dispatcher {worker_id} started\")\n",
|
||||
"\n",
|
||||
" while True:\n",
|
||||
" # get connections, based on layer 3 graph -> TODO: extend functionality\n",
|
||||
" # dirty read is not a problem here, modification will be picked up in the next iteration\n",
|
||||
" base_graph = current_state[\"layer_3_graph\"]\n",
|
||||
"\n",
|
||||
" # get connections, connections are edges in the graph\n",
|
||||
" connections = list(base_graph.edges())\n",
|
||||
"\n",
|
||||
" # get connection registry lock\n",
|
||||
" connection_registry_lock.acquire()\n",
|
||||
"\n",
|
||||
" # get connections that are not in the registry\n",
|
||||
" new_connections = [connection for connection in connections if connection not in connection_registry]\n",
|
||||
"\n",
|
||||
" # process connection updates since last update\n",
|
||||
" while not connection_update_done_queue.empty():\n",
|
||||
" # get connection from queue, wait at most update interval\n",
|
||||
" try:\n",
|
||||
" connection = connection_update_done_queue.get(timeout=update_interval_in_ms)\n",
|
||||
" except queue.Empty:\n",
|
||||
" break\n",
|
||||
" # update connection in registry, connection is tuple with connection and last updated\n",
|
||||
" connection_registry[connection][\"last_updated\"] = connection[1]\n",
|
||||
"\n",
|
||||
" # add new connections to registry\n",
|
||||
" for connection in new_connections:\n",
|
||||
" # add connection to registry, set last updated to None so that it is processed\n",
|
||||
" connection_registry[connection] = {\"last_updated\": None}\n",
|
||||
"\n",
|
||||
" # add connections to work queue, if last update is longer ago than the update interval or None\n",
|
||||
" for connection, connection_info in connection_registry.items():\n",
|
||||
" if connection_info[\"last_updated\"] is None or time.time() - connection_info[\n",
|
||||
" \"last_updated\"] > update_interval_in_ms:\n",
|
||||
" connection_update_queue.put(connection)\n",
|
||||
"\n",
|
||||
" # release connection registry lock\n",
|
||||
" connection_registry_lock.release()\n"
|
||||
],
|
||||
"id": "cca86765185e962c"
|
||||
},
|
||||
{
|
||||
"metadata": {},
|
||||
"cell_type": "code",
|
||||
"outputs": [],
|
||||
"execution_count": null,
|
||||
"source": [
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n"
|
||||
],
|
||||
"id": "b9b68e0a300b6b39"
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2024-02-23T14:00:52.816881Z",
|
||||
"start_time": "2024-02-23T14:00:52.644607Z"
|
||||
}
|
||||
},
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"\n",
|
||||
"n_workers = 1\n",
|
||||
"packet_workers = [threading.Thread(target=packet_worker, args=(processing_functions, worker_id))\n",
|
||||
" for worker_id in range(n_workers)]\n",
|
||||
"for worker_thread in packet_workers:\n",
|
||||
" worker_thread.daemon = True\n",
|
||||
"\n",
|
||||
"merger_thread = threading.Thread(target=merger, args=(merge_states,))\n",
|
||||
"merger_thread.daemon = True\n",
|
||||
"\n",
|
||||
"# start threads\n",
|
||||
"# for worker_thread in workers:\n",
|
||||
"# worker_thread.start()\n",
|
||||
"# \n",
|
||||
"# merger_thread.start()\n"
|
||||
],
|
||||
"id": "dde42074dc4deda9",
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "NameError",
|
||||
"evalue": "name 'threading' is not defined",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
|
||||
"\u001B[0;31mNameError\u001B[0m Traceback (most recent call last)",
|
||||
"Cell \u001B[0;32mIn[1], line 2\u001B[0m\n\u001B[1;32m 1\u001B[0m n_workers \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m1\u001B[39m\n\u001B[0;32m----> 2\u001B[0m workers \u001B[38;5;241m=\u001B[39m \u001B[43m[\u001B[49m\u001B[43mthreading\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mThread\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtarget\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mworker\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43margs\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mprocessing_functions\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mworker_id\u001B[49m\u001B[43m)\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 3\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;28;43;01mfor\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mworker_id\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;129;43;01min\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[38;5;28;43mrange\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mn_workers\u001B[49m\u001B[43m)\u001B[49m\u001B[43m]\u001B[49m\n\u001B[1;32m 4\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m worker_thread \u001B[38;5;129;01min\u001B[39;00m workers:\n\u001B[1;32m 5\u001B[0m worker_thread\u001B[38;5;241m.\u001B[39mdaemon \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n",
|
||||
"Cell \u001B[0;32mIn[1], line 2\u001B[0m, in \u001B[0;36m<listcomp>\u001B[0;34m(.0)\u001B[0m\n\u001B[1;32m 1\u001B[0m n_workers \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m1\u001B[39m\n\u001B[0;32m----> 2\u001B[0m workers \u001B[38;5;241m=\u001B[39m [\u001B[43mthreading\u001B[49m\u001B[38;5;241m.\u001B[39mThread(target\u001B[38;5;241m=\u001B[39mworker, args\u001B[38;5;241m=\u001B[39m(processing_functions, worker_id))\n\u001B[1;32m 3\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m worker_id \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mrange\u001B[39m(n_workers)]\n\u001B[1;32m 4\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m worker_thread \u001B[38;5;129;01min\u001B[39;00m workers:\n\u001B[1;32m 5\u001B[0m worker_thread\u001B[38;5;241m.\u001B[39mdaemon \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n",
|
||||
"\u001B[0;31mNameError\u001B[0m: name 'threading' is not defined"
|
||||
]
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 2
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython2",
|
||||
"version": "2.7.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@@ -0,0 +1,40 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=42", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "graph-intrusion-detection"
|
||||
version = "0.1.0"
|
||||
description = "A graph-based intrusion detection system."
|
||||
authors = [
|
||||
{ name = "Alex Blank", email = "alexblank@fastmail.com" }
|
||||
]
|
||||
license = { text = "MIT" }
|
||||
readme = "README.md"
|
||||
keywords = ["intrusion-detection", "graph-analysis", "network-security"]
|
||||
classifiers = [
|
||||
"Development Status :: 3 - Alpha",
|
||||
"Intended Audience :: Developers",
|
||||
"Topic :: Security",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Programming Language :: Python :: 3.11"
|
||||
]
|
||||
dependencies = [
|
||||
"keras>=2.15.0",
|
||||
"tqdm>=4.66.1",
|
||||
"networkx>=3.2.1",
|
||||
"numpy>=1.26.2",
|
||||
"pandas>=2.1.3",
|
||||
"scikit-learn>=1.3.2",
|
||||
"scipy>=1.11.4",
|
||||
"tensorflow>=2.15.0",
|
||||
"python-dotenv>=1.0.1",
|
||||
"kafka-python>=2.0.2",
|
||||
"redis>=5.0.1",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"matplotlib",
|
||||
]
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
,alex,alex-desktop,27.02.2024 15:37,file:///home/alex/.config/libreoffice/4;
|
||||
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 73 KiB |
@@ -0,0 +1,215 @@
|
||||
%! Author = alex
|
||||
%! Date = 2/26/24
|
||||
|
||||
% Preamble
|
||||
\documentclass[12pt]{article}
|
||||
|
||||
% Packages
|
||||
\usepackage{time}
|
||||
\usepackage{amsmath}
|
||||
\usepackage{graphicx}
|
||||
\usepackage{hyperref}
|
||||
\usepackage[backend=biber, style=authoryear, citestyle=authoryear-comp]{biblatex}
|
||||
\addbibresource{references.bib}
|
||||
|
||||
|
||||
\title{A Distributed System for Deep Packet Inspection and Event Detection}
|
||||
\author{Alexander Blank \\ Leipzig University - Chair of Mathematics and Computer Science}
|
||||
\date{\today}
|
||||
|
||||
% Document
|
||||
\begin{document}
|
||||
|
||||
\maketitle
|
||||
|
||||
\begin{abstract}
|
||||
This paper proposes a conceptual design for a distributed system aimed at enhancing deep packet
|
||||
inspection and event detection within large-scale network environments, a cornerstone of network security.
|
||||
It explores the integration of scalable, fault-tolerant architectures with modern technologies such as
|
||||
Apache Kafka for message brokering and Redis for state management, highlighting the potential for real-time
|
||||
analysis and anomaly detection.
|
||||
Through theoretical examination and design proposals, this work seeks to underscore the challenges and
|
||||
solutions associated with protecting expansive network infrastructures.
|
||||
While acknowledging the existence of similar systems, this project emphasizes the educational process of
|
||||
designing a network security solution, offering insights into system scalability, fault tolerance,
|
||||
and the practical application of theoretical knowledge.
|
||||
This endeavor not only contributes to the academic exploration of network security challenges but also
|
||||
enhances understanding of distributed system architectures' role in mitigating these challenges.
|
||||
\end{abstract}
|
||||
|
||||
|
||||
\section{Introduction}\label{sec:introduction}
|
||||
The escalating complexity and scale of modern networks pose significant challenges to traditional
|
||||
network security measures, necessitating the development of innovative, scalable solutions capable of real-time
|
||||
threat detection and analysis.
|
||||
This paper introduces a practical distributed system designed to meet these demands through deep packet inspection
|
||||
(DPI) and event detection.
|
||||
Utilizing a scalable, fault-tolerant architecture integrated with Apache Kafka for efficient message brokering and
|
||||
Redis for effective state management, the system offers a robust solution for monitoring and analyzing
|
||||
network traffic across large-scale environments.
|
||||
|
||||
Detailing the development and implementation phases, this work highlights the system's design principles,
|
||||
key functionalities, and its ability to adaptively scale, ensuring comprehensive network security coverage.
|
||||
By showcasing the system's deployment and its proficiency in detecting diverse network anomalies, the paper aims to
|
||||
contribute to the enhancement of network security mechanisms, addressing the critical need for systems that can
|
||||
evolve in tandem with the networks they protect.
|
||||
|
||||
Due to a length limitation, the paper focuses on the conceptual design and the system's key components, omitting
|
||||
detailed implementation and deployment aspects.
|
||||
More detailed information on the system's implementation and deployment can be found in the
|
||||
project's repository or provided by the author upon request.
|
||||
|
||||
|
||||
\section{Method}\label{sec:method}
|
||||
The distributed system is composed of at least 4 components, each with a specific role in the overall
|
||||
functionality of the system.
|
||||
Each component can be scaled horizontally without having to change the configuration of the system.
|
||||
The work will be distributed automatically to the available components, and the system will be able to
|
||||
handle the increased load without any manual intervention.
|
||||
|
||||
To actually use the results of the system, additional components can be added, that react to, or interact
|
||||
with changes in the network state or the individual nodes' analysis results.
|
||||
|
||||
Figure~\ref{fig:system_diagram} shows a diagram of the system's components and their interactions.
|
||||
% show a diagram of the system
|
||||
\begin{figure}[h]
|
||||
\centering
|
||||
\includegraphics[width=0.9\textwidth]{resources/system_diagram}
|
||||
\caption{The systems components and their interactions.}
|
||||
\label{fig:system_diagram}
|
||||
\end{figure}
|
||||
|
||||
\subsection{Packet Processor}\label{subsec:packet-processor}
|
||||
This component is responsible for analyzing batches of incoming packets,
|
||||
based on processing steps defined in the configuration of the system.
|
||||
|
||||
The system accepts network packets from a Kafka topic, which are then processed by the packet processors.
|
||||
Each processor takes in a predefined number of packets and processes them
|
||||
based on a list of processing steps, which are defined in the configuration of the system.
|
||||
Processing steps are defined as a list of functions, which are applied to each packet in the batch.
|
||||
The resulting information is written to a Kafka topic, which is then consumed by the state merger component.
|
||||
|
||||
\subsection{State Merger}\label{subsec:state-merger}
|
||||
This component is responsible for merging the information produced by the packet processor into a global state.
|
||||
Merging behaviour is defined in the configuration of the system.
|
||||
Each processing step also needs a merging function, that defines how the information produced by the processing
|
||||
step is merged into the global state.
|
||||
|
||||
A state merger will take in batches of processed packets, apply the respective merging functions to them and
|
||||
save the resulting global network state to a Redis database.
|
||||
A locking mechanism is used to ensure that no information is lost on concurrent writes to the database.
|
||||
|
||||
\subsection{Work Dispatcher}\label{subsec:work-dispatcher}
|
||||
This component is responsible for distributing the work resulting from the network state to a work queue.
|
||||
Work packages contain the node identity of the node that should be analyzed and the creation time to ensure that
|
||||
timeouts can clean up old packages that might otherwise clog up the work queue.
|
||||
An interval can be set, that defines how often each node should be analyzed.
|
||||
|
||||
Additionally to dispatching work packages, the work dispatcher also takes care of the results of said work packages
|
||||
by processing result messages from the result queue and updating the node registry accordingly.
|
||||
If a result does not come in after a given timeout, the work dispatcher will dispatch the work package again,
|
||||
taking into consideration any changes in network state that happened until then.
|
||||
|
||||
\subsection{Node Analysis}\label{subsec:node-analysis}
|
||||
This component is responsible for analyzing the network state of a given node and producing results based on
|
||||
configured analysis steps.
|
||||
The analysis steps are defined in the configuration of the system and contain a dataset function, that defines
|
||||
how the data for the analysis is retrieved from the global network state, and a model and scaler, that define
|
||||
the actual analysis model and, if needed, a scaling model, that gets applied to the extracted dataset before
|
||||
it is fed into the model.
|
||||
Depending on the model used, a model function needs to be given, that defines how to interact with it.
|
||||
For each analysis step, a interpretation function can be given, that will be applied to the models result to
|
||||
produce a usable interpretation or human-readable result.
|
||||
The set of results of each analysis step will then get written to the resul queue.
|
||||
|
||||
\subsection{Node Registry}\label{subsec:node-regsitry}
|
||||
The node registry contains all information analyzed by the system, such as the last time a node was analyzed
|
||||
and the result of each of the analysis steps.
|
||||
The registry can then be used by other components to react to these results, such as a recognition of an attack,
|
||||
that might trigger a countermeasure.
|
||||
|
||||
\subsection{Processing Step}\label{subsec:processing-step}
|
||||
As a proof of concept, a simple processing step is implemented, that converts the current network state based
|
||||
on the packets it receives into a graph representation of the network.
|
||||
This graph representation then serves as a base for further processing steps and the event detection component.
|
||||
The graph encompasses the nodes of the network as vertices and the connections between the nodes as edges.
|
||||
Each edge gets the data associated to the connection on a per connection and per second basis as edge attributes,
|
||||
based on the configuration of the source fields.
|
||||
Since this has the disadvantage of being limited to a resolution of one second and having a rather complex merging
|
||||
function, it should be noted that this is only a proof of concept and can be replaced by more sophisticated
|
||||
processing steps, which are tailored to the specific use case of the system.
|
||||
|
||||
\subsection{Analysis Models}
|
||||
There are two implemented models as a proof of concept for the event detection component.
|
||||
|
||||
The first model is a simple random forest classifier, which is trained on the dataset of network traffic
|
||||
developed in the lecture.
|
||||
Its purpose is to detect an incoming attack for any given node in the network.
|
||||
The attack is defined bye the attack column given by the dataset.
|
||||
On unseen test data, the model achieves a precision of 99.9\% and a recall of 99.9\% being trained on appropriaetly
|
||||
weighted data to reflect the huge class imbalance in the dataset.
|
||||
|
||||
|
||||
The second model is a neural network classifier aimed at detecting nodes that are behaving anomalously, i.e.\ nodes
|
||||
that are sources of attacks.
|
||||
The neural network is also trained on the dataset of network traffic developed in the lecture.
|
||||
This model achieves a precision of 99.9\% and a recall of 95.2\% on unseen test data.
|
||||
The lower recall is due to the fact that, as stated above, the dataset is highly imbalanced, and as a rather
|
||||
complex model architecture, the neural network is prone to overfitting on the majority class.
|
||||
Extending the dataset with more data from the minority class would likely improve the recall of the model.
|
||||
|
||||
However, the models are not the focus of this paper, and therefore, they are not discussed in detail and can
|
||||
be considered a proof of concept for the event detection component.
|
||||
They can easily be replaced by more sophisticated models, which are trained on more extensive datasets.
|
||||
|
||||
|
||||
\section{Future Work}
|
||||
Since the current implementation is only a proof of concept, there are many areas that can be improved and
|
||||
extended in the future.
|
||||
|
||||
The current implementation is based on a single network state, that needs to be written to and read from the
|
||||
Redis database on each update, limiting the scalability of the system.
|
||||
A shard-based approach can be used to distribute the network state across multiple distinct network states, which
|
||||
enables the global state to be distributed across multiple Redis instances, thus improving the scalability of the
|
||||
system.
|
||||
|
||||
The current processing approach is based on a network graph representation, which is updated in its entirety on each
|
||||
update of the network state, limiting the scalability of the merging step.
|
||||
To improve the performance of the system, switching to partial graph updates, which only update the parts of the
|
||||
graph that have changed, would be beneficial.
|
||||
Additionally, a binary merge method can be used to further distribute the work load on the merging step.
|
||||
Switching to different database architectures, such as a graph database for the network state and a time-series
|
||||
database for the network traffic, can also improve the performance of the system.
|
||||
|
||||
Interacting with Kafka and Redis require serialization and deserialization of the data, which can be a bottleneck
|
||||
and security risk.
|
||||
At the moment, python's pickle module is used for serialization and deserialization, which is not secure and
|
||||
needs to be replaced by a more secure serialization method such as Apache Avro upon deployment.
|
||||
|
||||
However, the most important future work is the extension of the system's analysis capabilities.
|
||||
At the moment, the models are heavily limited by the dataset used to train them.
|
||||
Extending the dataset with more data from the minority class would likely improve the recall, and thus the
|
||||
overall performance of the models.
|
||||
Different approaches to the analysis of the network state can be implemented, such as unsupervised learning on the
|
||||
graph representation for clustering or anomaly detection or transformer based approaches for capturing long-range dependencies
|
||||
in the traffic patterns of the network could prove beneficial.
|
||||
Additional representations of the network state can be implemented, that allow for a different perspective on
|
||||
the network state.
|
||||
|
||||
|
||||
\section{Conclusion}
|
||||
In summary, this paper has presented a conceptual design for a distributed system to enhance deep packet
|
||||
inspection and event detection within large-scale network environments.
|
||||
Despite its theoretical nature and the limitations posed by the current implementation,
|
||||
the proposed system demonstrates significant potential for real-time analysis, anomaly detection,
|
||||
and network security enhancement. Future work will focus on improving scalability, performance,
|
||||
and analysis capabilities, with the aim of developing a more sophisticated and efficient system.
|
||||
Through continuous improvement and adaptation, this project contributes to the ongoing evolution of network
|
||||
security mechanisms, ensuring they remain effective against the challenges of modern network infrastructures.
|
||||
|
||||
|
||||
|
||||
\printbibliography
|
||||
|
||||
|
||||
\end{document}
|
||||
@@ -0,0 +1,139 @@
|
||||
<mxfile host="app.diagrams.net" modified="2024-02-26T13:54:47.356Z" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 " etag="JpGQA6TeKbmNb_bQ45cY" version="23.1.6" type="device">
|
||||
<diagram name="Page-1" id="fKCp3sQInKXj2a9m8gdc">
|
||||
<mxGraphModel dx="1235" dy="685" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="3300" pageHeight="4681" math="0" shadow="0">
|
||||
<root>
|
||||
<mxCell id="0" />
|
||||
<mxCell id="1" parent="0" />
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-17" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="SmL7y99moSbFcQM2cvxb-1" target="SmL7y99moSbFcQM2cvxb-14">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<Array as="points">
|
||||
<mxPoint x="435" y="215" />
|
||||
<mxPoint x="540" y="215" />
|
||||
</Array>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-1" value="Processing Reults" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;direction=south;" vertex="1" parent="1">
|
||||
<mxGeometry x="360" y="110" width="150" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-13" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="SmL7y99moSbFcQM2cvxb-2" target="SmL7y99moSbFcQM2cvxb-9">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-2" value="Packet Sniffer" style="ellipse;shape=cloud;whiteSpace=wrap;html=1;" vertex="1" parent="1">
|
||||
<mxGeometry x="60" y="100" width="120" height="80" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-12" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="SmL7y99moSbFcQM2cvxb-5" target="SmL7y99moSbFcQM2cvxb-1">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<Array as="points">
|
||||
<mxPoint x="370" y="215" />
|
||||
<mxPoint x="400" y="215" />
|
||||
</Array>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-5" value="Packet Processor Workers" style="swimlane;whiteSpace=wrap;html=1;" vertex="1" parent="1">
|
||||
<mxGeometry x="220" y="260" width="200" height="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-7" value="" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="SmL7y99moSbFcQM2cvxb-5">
|
||||
<mxGeometry x="60" y="100" width="120" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-8" value="Packet Processor" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="SmL7y99moSbFcQM2cvxb-5">
|
||||
<mxGeometry x="30" y="80" width="120" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-11" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="SmL7y99moSbFcQM2cvxb-9" target="SmL7y99moSbFcQM2cvxb-5">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<Array as="points">
|
||||
<mxPoint x="260" y="215" />
|
||||
<mxPoint x="290" y="215" />
|
||||
</Array>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-9" value="Packets" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;direction=south;" vertex="1" parent="1">
|
||||
<mxGeometry x="220" y="110" width="80" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-14" value="State Merging Workers" style="swimlane;whiteSpace=wrap;html=1;" vertex="1" parent="1">
|
||||
<mxGeometry x="480" y="260" width="200" height="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-15" value="" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="SmL7y99moSbFcQM2cvxb-14">
|
||||
<mxGeometry x="60" y="100" width="120" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-16" value="State Merger" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="SmL7y99moSbFcQM2cvxb-14">
|
||||
<mxGeometry x="30" y="80" width="120" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-23" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="SmL7y99moSbFcQM2cvxb-18" target="SmL7y99moSbFcQM2cvxb-20">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<Array as="points">
|
||||
<mxPoint x="820" y="150" />
|
||||
</Array>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-18" value="Global State" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;" vertex="1" parent="1">
|
||||
<mxGeometry x="670" y="110" width="60" height="80" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-19" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1" source="SmL7y99moSbFcQM2cvxb-14" target="SmL7y99moSbFcQM2cvxb-18">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-28" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="SmL7y99moSbFcQM2cvxb-20" target="SmL7y99moSbFcQM2cvxb-27">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-20" value="Work Dispatching Workers" style="swimlane;whiteSpace=wrap;html=1;" vertex="1" parent="1">
|
||||
<mxGeometry x="720" y="260" width="200" height="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-21" value="" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="SmL7y99moSbFcQM2cvxb-20">
|
||||
<mxGeometry x="60" y="100" width="120" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-22" value="Work Dispatcher" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="SmL7y99moSbFcQM2cvxb-20">
|
||||
<mxGeometry x="30" y="80" width="120" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-31" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="SmL7y99moSbFcQM2cvxb-24" target="SmL7y99moSbFcQM2cvxb-30">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<Array as="points">
|
||||
<mxPoint x="700" y="640" />
|
||||
<mxPoint x="700" y="640" />
|
||||
</Array>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-24" value="Node Analysis Workers" style="swimlane;whiteSpace=wrap;html=1;" vertex="1" parent="1">
|
||||
<mxGeometry x="480" y="490" width="200" height="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-25" value="" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="SmL7y99moSbFcQM2cvxb-24">
|
||||
<mxGeometry x="60" y="100" width="120" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-26" value="Node Analysis Worker" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="SmL7y99moSbFcQM2cvxb-24">
|
||||
<mxGeometry x="20" y="80" width="130" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-29" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="SmL7y99moSbFcQM2cvxb-27" target="SmL7y99moSbFcQM2cvxb-24">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<Array as="points">
|
||||
<mxPoint x="700" y="550" />
|
||||
<mxPoint x="700" y="550" />
|
||||
</Array>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-27" value="Work Queue" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;direction=south;" vertex="1" parent="1">
|
||||
<mxGeometry x="745" y="520" width="150" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-32" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="SmL7y99moSbFcQM2cvxb-30" target="SmL7y99moSbFcQM2cvxb-20">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<Array as="points">
|
||||
<mxPoint x="970" y="640" />
|
||||
<mxPoint x="970" y="360" />
|
||||
</Array>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-30" value="Result Queue" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;direction=south;" vertex="1" parent="1">
|
||||
<mxGeometry x="745" y="610" width="150" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-33" value="Network Admin" style="shape=umlActor;verticalLabelPosition=bottom;verticalAlign=top;html=1;outlineConnect=0;" vertex="1" parent="1">
|
||||
<mxGeometry x="880" y="40" width="30" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-34" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1" source="SmL7y99moSbFcQM2cvxb-33" target="SmL7y99moSbFcQM2cvxb-18">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="SmL7y99moSbFcQM2cvxb-35" value="React to network state" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="SmL7y99moSbFcQM2cvxb-34">
|
||||
<mxGeometry x="0.2727" y="-6" relative="1" as="geometry">
|
||||
<mxPoint x="50" y="6" as="offset" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
</root>
|
||||
</mxGraphModel>
|
||||
</diagram>
|
||||
</mxfile>
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 78 KiB |
Binary file not shown.
Binary file not shown.
Binary file not shown.
+59
@@ -0,0 +1,59 @@
|
||||
import time
|
||||
import warnings
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from graph_based_intrusion_detection.utils import constants
|
||||
|
||||
|
||||
def run_analysis_step(nodes: list[str], analysis_step: dict, network_state: dict) -> dict:
|
||||
"""
|
||||
Run an analysis step on a dataset
|
||||
:param nodes: nodes to analyze
|
||||
:param analysis_step: dictionary with analysis step configuration
|
||||
:param network_state: network state dictionary
|
||||
:return: analysis results
|
||||
"""
|
||||
|
||||
# extract network graph from network state
|
||||
network_state_data = network_state[analysis_step["network_state_key"]]
|
||||
|
||||
# apply dataset function to extract dataset for node
|
||||
datasets = [analysis_step["dataset_function"](network_state_data, node) for node in nodes]
|
||||
|
||||
# remove identifier columns from datasets
|
||||
datasets = [dataset.drop(columns=analysis_step["dataset_identifiers"]) for dataset in datasets]
|
||||
|
||||
# scale datasets
|
||||
scaled_datasets = [analysis_step["scaler"].transform(dataset) for dataset in datasets]
|
||||
|
||||
# apply preprocessing function, if given
|
||||
if "preprocessing_function" in analysis_step:
|
||||
scaled_datasets = [analysis_step["preprocessing_function"](scaled_dataset) for scaled_dataset in
|
||||
scaled_datasets]
|
||||
|
||||
# combine datasets, get indices to split them later
|
||||
combined_dataset = np.concatenate(scaled_datasets)
|
||||
split_indices = list()
|
||||
for dataset in scaled_datasets:
|
||||
split_indices.append(len(dataset) + split_indices[-1] if split_indices else len(dataset))
|
||||
|
||||
# predict and interpret results
|
||||
prediction_function = analysis_step["predict_function"]
|
||||
if constants.SURPRESS_WARNINGS:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
predictions = prediction_function(combined_dataset)
|
||||
else:
|
||||
predictions = prediction_function(combined_dataset)
|
||||
|
||||
# split predictions
|
||||
predictions = np.split(predictions, split_indices[:-1])
|
||||
|
||||
results = [analysis_step["interpretation_function"](prediction) for prediction in predictions]
|
||||
|
||||
# compose results with node names to dict
|
||||
result_dict = {node: result for node, result in zip(nodes, results)}
|
||||
|
||||
return result_dict
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
|
||||
|
||||
def extract_node_connections(node, edges):
|
||||
"""
|
||||
Extract the connections of a node from a list of edges
|
||||
:param node: node identifier
|
||||
:param edges: list of edges
|
||||
:return: list of connections
|
||||
"""
|
||||
connections = list()
|
||||
for edge in edges:
|
||||
if edge[0] == node:
|
||||
connections.append(edge[1])
|
||||
else:
|
||||
connections.append(edge[0])
|
||||
|
||||
return connections
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
def interpret_prediction_result(result):
|
||||
return np.max(result)
|
||||
+255
@@ -0,0 +1,255 @@
|
||||
import networkx as nx
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def get_info_for_node(graph: nx.Graph, node: str):
|
||||
"""
|
||||
Get info for a node
|
||||
:param graph: graph to analyze
|
||||
:param node: node to analyze
|
||||
:return: info for the node
|
||||
"""
|
||||
edges = list(graph.edges(node, data=True))
|
||||
return edges
|
||||
|
||||
|
||||
def extract_connections(graph: nx.Graph) -> list:
|
||||
"""
|
||||
Extract connections from a graph
|
||||
:param graph: graph to analyze
|
||||
:return: list of connections
|
||||
"""
|
||||
connections = []
|
||||
for src, dest, data in graph.edges(data=True):
|
||||
connections.append((src, dest, data))
|
||||
return connections
|
||||
|
||||
|
||||
def create_dataset_for_connection(connection_edge_data: dict,
|
||||
per_second_attributes: list[str] = None,
|
||||
per_connection_attributes: list[str] = None,
|
||||
first_second: int = None,
|
||||
last_second: int = None) -> pd.DataFrame:
|
||||
"""
|
||||
Create a dataset for a connection based on the graph edge data
|
||||
:param connection_edge_data: graph edge data for the connection
|
||||
:param per_second_attributes: attributes to extract per second
|
||||
:param per_connection_attributes: attributes to extract per connection
|
||||
:param first_second: first second to include in the dataset
|
||||
:param last_second: last second to include in the dataset
|
||||
:return: dataset for the connection
|
||||
"""
|
||||
|
||||
|
||||
def create_dataset_for_node(graph: nx.Graph,
|
||||
node: str,
|
||||
per_second_attributes: list[str] = None,
|
||||
per_connection_attributes: list[str] = None,
|
||||
first_second: int = None,
|
||||
last_second: int = None) -> pd.DataFrame:
|
||||
"""
|
||||
Create a dataset for a connection
|
||||
:param graph: graph to analyze
|
||||
:param node: name of the node
|
||||
:param per_second_attributes: attributes to extract per second
|
||||
:param per_connection_attributes: attributes to extract per connection
|
||||
:param first_second: first second to include in the dataset
|
||||
:param last_second: last second to include in the dataset
|
||||
:return: dataset for the connection
|
||||
"""
|
||||
|
||||
# get incoming and outgoing edges for node
|
||||
incoming = list(graph.in_edges(node, data=True))
|
||||
outgoing = list(graph.out_edges(node, data=True))
|
||||
|
||||
out_packet_seconds = list()
|
||||
for edge in outgoing:
|
||||
out_packet_seconds.extend(list(edge[2]["packets_per_second"].keys()))
|
||||
|
||||
out_packet_seconds = sorted(list(set(out_packet_seconds)))
|
||||
|
||||
in_packet_seconds = list()
|
||||
for edge in incoming:
|
||||
in_packet_seconds.extend(list(edge[2]["packets_per_second"].keys()))
|
||||
|
||||
in_packet_seconds = sorted(list(set(in_packet_seconds)))
|
||||
|
||||
# filter out seconds, if first_second or last_second is given
|
||||
if first_second is not None or last_second is not None:
|
||||
if first_second is None:
|
||||
first_second = min(out_packet_seconds + in_packet_seconds)
|
||||
if last_second is None:
|
||||
last_second = max(out_packet_seconds + in_packet_seconds)
|
||||
|
||||
in_packet_seconds = [second for second in in_packet_seconds if first_second <= second <= last_second]
|
||||
out_packet_seconds = [second for second in out_packet_seconds if first_second <= second <= last_second]
|
||||
|
||||
# extract per connection values
|
||||
per_connection_values_in = dict()
|
||||
for src, dest, data in incoming:
|
||||
for attribute in per_connection_attributes:
|
||||
if attribute in data:
|
||||
if isinstance(data[attribute], list):
|
||||
# when list, use length
|
||||
per_connection_values_in[f"{attribute}_in"] = [len(data[attribute])] * len(in_packet_seconds)
|
||||
else:
|
||||
per_connection_values_in[f"{attribute}_in"] = [data[attribute]] * len(in_packet_seconds)
|
||||
else:
|
||||
per_connection_values_in[f"{attribute}_in"] = [0] * len(in_packet_seconds)
|
||||
|
||||
if len(incoming) == 0:
|
||||
per_connection_values_in = {f"{attribute}_in": [0] * len(in_packet_seconds) for attribute in
|
||||
per_connection_attributes}
|
||||
|
||||
per_connection_values_out = dict()
|
||||
for src, dest, data in outgoing:
|
||||
for attribute in per_connection_attributes:
|
||||
if attribute in data:
|
||||
if isinstance(data[attribute], list):
|
||||
# when list, use length
|
||||
per_connection_values_out[f"{attribute}_out"] = [len(data[attribute])] * len(out_packet_seconds)
|
||||
else:
|
||||
per_connection_values_out[f"{attribute}_out"] = [data[attribute]] * len(out_packet_seconds)
|
||||
else:
|
||||
per_connection_values_out[f"{attribute}_out"] = [0] * len(out_packet_seconds)
|
||||
|
||||
if len(outgoing) == 0:
|
||||
per_connection_values_out = {f"{attribute}_out": [0] * len(out_packet_seconds) for attribute in
|
||||
per_connection_attributes}
|
||||
|
||||
# extract per second value for incoming connections
|
||||
per_second_values_in = list()
|
||||
for second in in_packet_seconds:
|
||||
values_this_second = dict()
|
||||
for src, dest, data in incoming:
|
||||
for attribute in per_second_attributes:
|
||||
# extract attribute values for the second, if they exist, else set to 0
|
||||
if attribute in data:
|
||||
attribute_values = data[attribute]
|
||||
if second in attribute_values:
|
||||
if attribute not in values_this_second:
|
||||
values_this_second[f"{attribute}_in"] = attribute_values[second]
|
||||
else:
|
||||
values_this_second[f"{attribute}_in"] += attribute_values[second]
|
||||
else:
|
||||
values_this_second[f"{attribute}_in"] = 0
|
||||
else:
|
||||
values_this_second[f"{attribute}_in"] = 0
|
||||
|
||||
# add to per second values
|
||||
per_second_values_in.append(values_this_second)
|
||||
|
||||
# still add attributes to dataset, in case of no incoming packets
|
||||
if len(in_packet_seconds) == 0:
|
||||
per_second_values_in = [{f"{attribute}_in": 0 for attribute in per_second_attributes}]
|
||||
|
||||
# extract per second values for outgoing connections
|
||||
per_second_values_out = list()
|
||||
for second in out_packet_seconds:
|
||||
values_this_second = dict()
|
||||
for src, dest, data in outgoing:
|
||||
for attribute in per_second_attributes:
|
||||
# extract attribute values for the second, if they exist, else set to 0
|
||||
if attribute in data:
|
||||
attribute_values = data[attribute]
|
||||
if second in attribute_values:
|
||||
if attribute not in values_this_second:
|
||||
values_this_second[f"{attribute}_out"] = attribute_values[second]
|
||||
else:
|
||||
values_this_second[f"{attribute}_out"] += attribute_values[second]
|
||||
else:
|
||||
values_this_second[f"{attribute}_out"] = 0
|
||||
else:
|
||||
values_this_second[f"{attribute}_out"] = 0
|
||||
|
||||
# add to per second values
|
||||
per_second_values_out.append(values_this_second)
|
||||
|
||||
# still add attributes to dataset, in case of no outgoing packets
|
||||
if len(out_packet_seconds) == 0:
|
||||
per_second_values_out = [{f"{attribute}_out": 0 for attribute in per_second_attributes}]
|
||||
|
||||
"""
|
||||
Process meta data for the node, such as time offsets between packets
|
||||
"""
|
||||
|
||||
# get time diff to last outgoing packet
|
||||
diff_to_last_outgoing = [0] + np.diff(out_packet_seconds).tolist()
|
||||
# get time diff to last incoming packet
|
||||
diff_to_last_incoming = [0] + np.diff(in_packet_seconds).tolist()
|
||||
|
||||
# get the second where each incoming connection was first seen by extracting it from the first packet second
|
||||
incoming_first_seen = [list(data["packets_per_second"].keys())[0] for src, dest, data in incoming]
|
||||
|
||||
# get new incoming connections per second by adding the first_seen occurrences for each second
|
||||
incoming_new_per_second = [sum([1 for first_seen in incoming_first_seen if first_seen == second])
|
||||
for second in in_packet_seconds]
|
||||
|
||||
# get the total number of incoming connection made per second, is sum of new connections per second until current index
|
||||
total_incoming_per_second = [sum(incoming_new_per_second[:index + 1]) for index in range(len(in_packet_seconds))]
|
||||
|
||||
# get the second where each outgoing connection was first seen by extracting it from the first packet second
|
||||
outgoing_first_seen = [list(data["packets_per_second"].keys())[0] for src, dest, data in outgoing]
|
||||
|
||||
# get new outgoing connections per second by adding the first_seen occurrences for each second
|
||||
outgoing_new_per_second = [sum([1 for first_seen in outgoing_first_seen if first_seen == second])
|
||||
for second in out_packet_seconds]
|
||||
|
||||
# get the total number of outgoing connection made per second
|
||||
total_outgoing_per_second = [sum(outgoing_new_per_second[:index + 1]) for index in range(len(out_packet_seconds))]
|
||||
|
||||
# get total incoming ports
|
||||
# incoming_ports = list()
|
||||
# for src, dest, data in incoming:
|
||||
# incoming_ports = [port for port in data["src_ports"] if port not in incoming_ports]
|
||||
#
|
||||
# if len(incoming_ports) == 0:
|
||||
# incoming_ports = [0]
|
||||
#
|
||||
# # get total outgoing ports
|
||||
# outgoing_ports = list()
|
||||
# for src, dest, data in outgoing:
|
||||
# outgoing_ports = [port for port in data["dst_ports"] if port not in outgoing_ports]
|
||||
#
|
||||
# if len(outgoing_ports) == 0:
|
||||
# outgoing_ports = [0]
|
||||
|
||||
per_second_incoming_df = pd.DataFrame(per_second_values_in, index=in_packet_seconds)
|
||||
incoming_data = {
|
||||
"diff_to_last_incoming": diff_to_last_incoming,
|
||||
"new_incoming_at_second": incoming_new_per_second,
|
||||
"total_incoming_at_second": total_incoming_per_second,
|
||||
} | per_connection_values_in
|
||||
# set index to the packet seconds
|
||||
per_node_incoming_df = pd.DataFrame(incoming_data, index=in_packet_seconds)
|
||||
incoming_df = pd.merge(per_node_incoming_df, per_second_incoming_df, how="outer", left_index=True,
|
||||
right_index=True)
|
||||
|
||||
per_second_outgoing_df = pd.DataFrame(per_second_values_out, index=out_packet_seconds)
|
||||
outgoing_data = {
|
||||
"diff_to_last_outgoing": diff_to_last_outgoing,
|
||||
"new_outgoing_at_second": outgoing_new_per_second,
|
||||
"total_outgoing_at_second": total_outgoing_per_second,
|
||||
} | per_connection_values_out
|
||||
# set index to the packet seconds
|
||||
per_node_outgoing_df = pd.DataFrame(outgoing_data, index=out_packet_seconds)
|
||||
outgoing_df = pd.merge(per_node_outgoing_df, per_second_outgoing_df, how="outer", left_index=True,
|
||||
right_index=True)
|
||||
|
||||
# merge incoming and outgoing data on packet seconds
|
||||
data = pd.merge(incoming_df, outgoing_df, how="outer", left_index=True, right_index=True)
|
||||
|
||||
# fill missing values with last set value (forward fill)
|
||||
data = data.ffill()
|
||||
|
||||
# fill missing values with 0, there can still be missing value in the first rows
|
||||
data = data.fillna(0)
|
||||
|
||||
# parse all columns as int
|
||||
data = data.astype(int)
|
||||
|
||||
# add identifier for the node
|
||||
data["node"] = node
|
||||
return data
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
import networkx as nx
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
|
||||
def extract_to_graph(input_graph: nx.Graph,
|
||||
packet: dict | pd.Series,
|
||||
source_field: str,
|
||||
destination_field: str,
|
||||
per_packet_fields: list[str],
|
||||
per_second_fields: list[str],
|
||||
timedelta_field: str):
|
||||
"""
|
||||
Extract the given fields from the packet and add them to the graph
|
||||
:param input_graph: graph to update
|
||||
:param packet: packet to analyze
|
||||
:param source_field: source field
|
||||
:param destination_field: destination field
|
||||
:param per_packet_fields: fields to extract per packet
|
||||
:param per_second_fields: fields to extract per second
|
||||
:param timedelta_field: field containing the timedelta to the start of the capture
|
||||
:return: updated graph
|
||||
"""
|
||||
# get timedelta to start of capture
|
||||
timedelta = packet[timedelta_field]
|
||||
|
||||
# get current second
|
||||
try:
|
||||
second = int(timedelta.total_seconds())
|
||||
except ValueError:
|
||||
# unusable packet
|
||||
return input_graph
|
||||
|
||||
# get src and destination address
|
||||
src = packet[source_field]
|
||||
dst = packet[destination_field]
|
||||
|
||||
if pd.isna(src) or pd.isna(dst):
|
||||
# unusable packet
|
||||
return input_graph
|
||||
|
||||
# add nodes, if not already present
|
||||
if not input_graph.has_node(src):
|
||||
input_graph.add_node(src)
|
||||
if not input_graph.has_node(dst):
|
||||
input_graph.add_node(dst)
|
||||
|
||||
# check content of packet to determine layer
|
||||
if packet["tcp/tcp.srcport"] is not np.nan:
|
||||
min_layer = 4
|
||||
elif packet["ip/ip.src"] is not np.nan:
|
||||
min_layer = 3
|
||||
elif packet["eth/eth.src"] is not np.nan:
|
||||
min_layer = 2
|
||||
else:
|
||||
min_layer = 1
|
||||
|
||||
# add edge, if not already present
|
||||
if not input_graph.has_edge(src, dst):
|
||||
input_graph.add_edge(src, dst)
|
||||
|
||||
# add per packet fields
|
||||
for field in per_packet_fields:
|
||||
if field["min_layer"] <= min_layer:
|
||||
value = field["base_value_function"](packet)
|
||||
if value is not None:
|
||||
input_graph[src][dst][field["name"]] = value
|
||||
|
||||
# add per second fields
|
||||
for field in per_second_fields:
|
||||
if field["min_layer"] <= min_layer:
|
||||
value = field["base_value_function"](packet)
|
||||
if value is not None:
|
||||
input_graph[src][dst][field["name"]] = {second: value}
|
||||
|
||||
else:
|
||||
|
||||
# update per packet fields
|
||||
for field in per_packet_fields:
|
||||
if field["min_layer"] <= min_layer:
|
||||
value = field["update_function"](input_graph[src][dst][field["name"]], packet)
|
||||
if value is not None:
|
||||
input_graph[src][dst][field["name"]] = value
|
||||
|
||||
# update per second fields
|
||||
for field in per_second_fields:
|
||||
if field["name"] in input_graph[src][dst]:
|
||||
if second not in input_graph[src][dst][field["name"]]:
|
||||
if field["min_layer"] <= min_layer:
|
||||
value = field["base_value_function"](packet)
|
||||
if value is not None:
|
||||
input_graph[src][dst][field["name"]][second] = value
|
||||
else:
|
||||
if field["min_layer"] <= min_layer:
|
||||
value = field["update_function"](input_graph[src][dst][field["name"]][second], packet)
|
||||
if value is not None:
|
||||
input_graph[src][dst][field["name"]][second] = value
|
||||
else:
|
||||
if field["min_layer"] <= min_layer:
|
||||
value = field["base_value_function"](packet)
|
||||
if value is not None:
|
||||
input_graph[src][dst][field["name"]] = {second: value}
|
||||
|
||||
return input_graph
|
||||
|
||||
|
||||
def update_layer_3_graph(packet: pd.Series,
|
||||
state: dict,
|
||||
per_packet_fields: list,
|
||||
per_second_fields: list,
|
||||
timedelta_field: str) -> nx.Graph:
|
||||
"""
|
||||
Update the layer 3 graph
|
||||
:param packet: packet to analyze
|
||||
:param state: current state of the model
|
||||
:param per_packet_fields: fields to extract per packet
|
||||
:param per_second_fields: fields to extract per second
|
||||
:param timedelta_field: field containing the timedelta to the start of the capture
|
||||
:return: updated state
|
||||
"""
|
||||
layer_3_graph = state if state is not None else nx.DiGraph()
|
||||
|
||||
layer_3_graph = extract_to_graph(layer_3_graph, packet,
|
||||
"ip/ip.src",
|
||||
"ip/ip.dst",
|
||||
per_packet_fields,
|
||||
per_second_fields,
|
||||
timedelta_field)
|
||||
|
||||
return layer_3_graph
|
||||
|
||||
|
||||
def update_layer_2_graph(packet: pd.Series,
|
||||
state: dict,
|
||||
per_packet_fields: list,
|
||||
per_second_fields: list,
|
||||
timedelta_field: str) -> nx.Graph:
|
||||
"""
|
||||
Update the layer 2 graph
|
||||
:param packet: packet to analyze
|
||||
:param state: current state of the model
|
||||
:param per_packet_fields: fields to extract per packet
|
||||
:param per_second_fields: fields to extract per second
|
||||
:param timedelta_field: field containing the timedelta to the start of the capture
|
||||
:return: updated state
|
||||
"""
|
||||
layer_2_graph = state if state is not None else nx.DiGraph()
|
||||
|
||||
layer_2_graph = extract_to_graph(layer_2_graph, packet,
|
||||
"eth/eth.src",
|
||||
"eth/eth.dst",
|
||||
per_packet_fields,
|
||||
per_second_fields,
|
||||
timedelta_field)
|
||||
|
||||
return layer_2_graph
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def update_port_list(packet: dict | pd.Series, current_port_list: list, port_field: str) -> list:
|
||||
if pd.isna(packet[port_field]):
|
||||
return current_port_list
|
||||
else:
|
||||
port = int(packet[port_field])
|
||||
if port not in current_port_list:
|
||||
current_port_list.append(port)
|
||||
return current_port_list
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
from graph_based_intrusion_detection.utils.state_merging import merge_states
|
||||
|
||||
|
||||
def multi_process_packets(packets: pd.DataFrame,
|
||||
global_parameterless_processing_function: callable,
|
||||
merging_function: callable,
|
||||
num_processes: int = 4,
|
||||
chunk_size_factor: float = 1,
|
||||
chunk_size: int = None) -> dict:
|
||||
"""
|
||||
Process a set of packets using multiple processes
|
||||
:param packets: packets to process
|
||||
:param global_parameterless_processing_function: function to apply to the packets, must be parameterless and global
|
||||
:param merging_function: function to merge the states
|
||||
:param num_processes: number of processes to use
|
||||
:param chunk_size_factor: factor to use to determine the chunk size, cannot be used with chunk_size, can reduce memory usage
|
||||
:param chunk_size: size of chunks to split the packets into, if None, the packets are split into num_processes chunks, cannot be used with chunk_size_factor
|
||||
:return: updated state
|
||||
"""
|
||||
|
||||
# chunk packets
|
||||
if chunk_size is not None:
|
||||
packet_chunks = np.array_split(packets, len(packets) // chunk_size)
|
||||
else:
|
||||
packet_chunks = np.array_split(packets, int(num_processes * (1 / chunk_size_factor)))
|
||||
|
||||
# create executor with tqdm
|
||||
with ProcessPoolExecutor(max_workers=num_processes) as executor:
|
||||
states = list(
|
||||
tqdm(executor.map(global_parameterless_processing_function, packet_chunks), total=len(packet_chunks)))
|
||||
|
||||
# merge states
|
||||
merged_state = states[0]
|
||||
for state in states[1:]:
|
||||
merged_state = merging_function(merged_state, state)
|
||||
return merged_state
|
||||
|
||||
|
||||
def process_packet(packet: dict | pd.Series,
|
||||
state: dict,
|
||||
processing_functions: list[callable]) -> dict:
|
||||
"""
|
||||
Digest a packet and update the state
|
||||
:param packet: packet to digest
|
||||
:param state: state of the model
|
||||
:param processing_functions: functions to apply to the packet
|
||||
:return: updated state
|
||||
"""
|
||||
|
||||
for processing_function in processing_functions:
|
||||
state = processing_function(packet, state)
|
||||
|
||||
return state
|
||||
|
||||
|
||||
def process_packets(packets: pd.DataFrame | list,
|
||||
processing_functions: list[callable],
|
||||
state=None,
|
||||
verbose: bool = False) -> dict:
|
||||
"""
|
||||
Process a set of packets
|
||||
:param packets: packets to process
|
||||
:param processing_functions: functions to apply to the packets
|
||||
:param state: state of the model
|
||||
:param verbose: if True, print progress
|
||||
:return: updated state
|
||||
"""
|
||||
|
||||
iterator = packets.to_dict("records") if isinstance(packets, pd.DataFrame) else packets
|
||||
if verbose:
|
||||
for packet in tqdm(iterator, total=len(packets)):
|
||||
state = process_packet(packet, state, processing_functions)
|
||||
else:
|
||||
for packet in iterator:
|
||||
state = process_packet(packet, state, processing_functions)
|
||||
|
||||
return state
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
import time
|
||||
|
||||
import redis
|
||||
|
||||
from graph_based_intrusion_detection.utils import constants
|
||||
from graph_based_intrusion_detection.utils import config
|
||||
from graph_based_intrusion_detection.utils.logging import create_logger
|
||||
from graph_based_intrusion_detection.utils.network_transport_utils import deserialize, serialize
|
||||
|
||||
|
||||
def start_network_visualizer(worker_id: str):
|
||||
"""
|
||||
Process that visualizes info about the network state
|
||||
:param worker_id: network id
|
||||
:return: None
|
||||
"""
|
||||
|
||||
logger = create_logger(f"Network-Visualizer {worker_id}")
|
||||
logger.info(f"Worker started")
|
||||
|
||||
redis_host = redis.Redis(host='localhost', port=6379, db=0)
|
||||
while True:
|
||||
node_registry_raw = redis_host.get(constants.NODE_REGISTRY_KEY)
|
||||
if node_registry_raw is None:
|
||||
print("Node registry not found in redis, retrying on next iteration")
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
# clear output
|
||||
os.system('clear')
|
||||
|
||||
node_registry = deserialize(node_registry_raw)
|
||||
|
||||
for node, node_data in node_registry.items():
|
||||
print(f"Node {node}, last updated: {node_data['last_updated']}")
|
||||
if "results" in node_data:
|
||||
for key, value in node_data["results"].items():
|
||||
print(f"\t{key}: {value}", end=",")
|
||||
else:
|
||||
print(f"\tNo results for node {node}")
|
||||
print()
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
worker_id = str(uuid.uuid4())
|
||||
else:
|
||||
worker_id = sys.argv[1]
|
||||
|
||||
start_network_visualizer(worker_id)
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
import sys
|
||||
import joblib
|
||||
import uuid
|
||||
import time
|
||||
|
||||
from kafka import KafkaConsumer, KafkaProducer
|
||||
import redis
|
||||
|
||||
from graph_based_intrusion_detection.utils import constants
|
||||
from graph_based_intrusion_detection.utils import config
|
||||
from graph_based_intrusion_detection.utils.logging import create_logger
|
||||
from graph_based_intrusion_detection.utils.network_transport_utils import deserialize, serialize
|
||||
from graph_based_intrusion_detection.graph_processing.data_extraction import create_dataset_for_node
|
||||
from graph_based_intrusion_detection.analysis.analysis_steps import run_analysis_step
|
||||
|
||||
|
||||
def start_connection_analysis(worker_id: str) -> None:
|
||||
"""
|
||||
Starts the connection analysis worker
|
||||
:param worker_id: id of the worker
|
||||
:return: None
|
||||
"""
|
||||
|
||||
logger = create_logger(f"Connection-Analysis {worker_id}")
|
||||
logger.info(f"Connection-Analysis {worker_id} started")
|
||||
|
||||
# load analysis models
|
||||
logger.info(f"Loading analysis models")
|
||||
analysis_models = config.ANALYSIS_STEPS
|
||||
for step in analysis_models:
|
||||
# load model and scaler for analysis step
|
||||
scaler = joblib.load(step["scaler_path"])
|
||||
model = joblib.load(step["model_path"])
|
||||
prediction_function = eval(f"model.{step['predict_function']}")
|
||||
|
||||
# append model and scaler to step
|
||||
step["scaler"] = scaler
|
||||
step["model"] = model
|
||||
step["predict_function"] = prediction_function
|
||||
|
||||
logger.info(f"Loaded model {step['name']}")
|
||||
|
||||
# connect to connection update queue
|
||||
node_update_consumer = KafkaConsumer(
|
||||
constants.NODE_ANALYSIS_TOPIC,
|
||||
bootstrap_servers='localhost:9092',
|
||||
auto_offset_reset='earliest')
|
||||
node_update_result_producer = KafkaProducer(
|
||||
bootstrap_servers="localhost:9092",
|
||||
)
|
||||
|
||||
# connect to redis
|
||||
redis_client = redis.StrictRedis(host="localhost", port=6379, db=0)
|
||||
|
||||
while True:
|
||||
# get batch of connections from kafka to analyze
|
||||
node_batch = list()
|
||||
while len(node_batch) < constants.NODE_ANALYSIS_BATCH_SIZE:
|
||||
node_messages = node_update_consumer.poll(timeout_ms=constants.NODE_ANALYSIS_POLL_TIMEOUT_MS,
|
||||
max_records=constants.NODE_ANALYSIS_BATCH_SIZE - len(
|
||||
node_batch))
|
||||
|
||||
if len(node_messages) == 0:
|
||||
break
|
||||
|
||||
for _, messages in node_messages.items():
|
||||
for message in messages:
|
||||
node = deserialize(message.value)
|
||||
node_batch.append(node)
|
||||
|
||||
if len(node_batch) == 0:
|
||||
time.sleep(0.1)
|
||||
continue
|
||||
|
||||
# remove update message older than the timeout
|
||||
node_batch = [node["node"] for node in node_batch if
|
||||
(time.time() - node["time"]) < constants.NODE_ANALYSIS_TIMEOUT_MS / 1000]
|
||||
|
||||
if len(node_batch) == 0:
|
||||
continue
|
||||
|
||||
# get network state from redis, dirty read is not really a problem since updates will be processed in the
|
||||
# next iteration
|
||||
network_state_raw = redis_client.get(constants.NETWORK_STATE_KEY)
|
||||
|
||||
if network_state_raw is None:
|
||||
logger.critical(f"Network state not found in redis, retrying on next iteration")
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
network_state = deserialize(network_state_raw)
|
||||
|
||||
# run all analysis steps
|
||||
logger.info(f"Analyzing {len(node_batch)} nodes")
|
||||
step_results = {step["name"]: run_analysis_step(node_batch, step, network_state) for step in analysis_models}
|
||||
|
||||
# compose result dict for each node and send result to kafka topic
|
||||
for node in node_batch:
|
||||
node_results = {step_name: step_results[node] for step_name, step_results in step_results.items()}
|
||||
|
||||
# compose message with node, time and results
|
||||
result_message = {
|
||||
"node": node,
|
||||
"time": time.time(),
|
||||
"results": node_results
|
||||
}
|
||||
|
||||
# send result to kafka
|
||||
node_update_result_producer.send(constants.NODE_ANALYSIS_RESULTS_TOPIC, serialize(result_message))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
worker_id = str(uuid.uuid4())
|
||||
print(f"Worker id not provided, using {worker_id}")
|
||||
else:
|
||||
worker_id = sys.argv[1]
|
||||
|
||||
start_connection_analysis(worker_id)
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from kafka import KafkaConsumer, KafkaProducer
|
||||
import pandas as pd
|
||||
|
||||
from graph_based_intrusion_detection.utils import constants
|
||||
from graph_based_intrusion_detection.utils import config
|
||||
from graph_based_intrusion_detection.packet_processing.processing_functions import process_packet, process_packets
|
||||
from graph_based_intrusion_detection.utils.logging import create_logger
|
||||
from graph_based_intrusion_detection.utils.network_transport_utils import deserialize, serialize
|
||||
|
||||
|
||||
def start_packet_processing(worker_id: str,
|
||||
max_batch_size: int = constants.PACKET_CONSUMER_BATCH_SIZE):
|
||||
"""
|
||||
Worker thread, processes packets from the packet queue and puts the resulting state into the state queue
|
||||
:param worker_id: id of the worker
|
||||
:param max_batch_size: maximum batch size
|
||||
:return: None
|
||||
"""
|
||||
|
||||
logger = create_logger(f"Packet-Worker {worker_id}")
|
||||
logger.info(f"Worker started")
|
||||
|
||||
packet_consumer = KafkaConsumer(constants.PACKET_TOPIC_NAME,
|
||||
bootstrap_servers='localhost:9092',
|
||||
auto_offset_reset='earliest')
|
||||
|
||||
state_producer = KafkaProducer(
|
||||
bootstrap_servers='localhost:9092', )
|
||||
|
||||
logger.info(f"Initialized, waiting for packets")
|
||||
while True:
|
||||
# consume batch of packets from kafka
|
||||
batch = list()
|
||||
while len(batch) < max_batch_size:
|
||||
packet_messages = packet_consumer.poll(timeout_ms=constants.PACKET_POLL_TIMEOUT_MS,
|
||||
max_records=max_batch_size - len(batch))
|
||||
|
||||
if len(packet_messages) == 0:
|
||||
break
|
||||
|
||||
for _, messages in packet_messages.items():
|
||||
for message in messages:
|
||||
packet = deserialize(message.value)
|
||||
batch.append(packet)
|
||||
|
||||
if len(batch) == 0:
|
||||
time.sleep(0.1)
|
||||
continue
|
||||
|
||||
logger.info(f"Processing {len(batch)} packets")
|
||||
|
||||
# process batch
|
||||
if len(batch) > 0:
|
||||
# process batch with all processing steps
|
||||
logger.info(f"Processing {len(batch)} packets in {len(config.PROCESSING_STEPS)} processing steps")
|
||||
|
||||
processing_step_results = dict()
|
||||
for processing_step in config.PROCESSING_STEPS:
|
||||
current_results = process_packets(batch, processing_step["processing_functions"])
|
||||
processing_step_results[processing_step["name"]] = current_results
|
||||
|
||||
serialized_state = serialize(processing_step_results)
|
||||
state_producer.send(constants.PROCESSED_STATE_TOPIC_NAME, value=serialized_state)
|
||||
logger.info(f"Processed {len(batch)} packets")
|
||||
else:
|
||||
logger.info(f"No packets to process")
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# get worker id from arguments
|
||||
if len(sys.argv) != 2:
|
||||
# assign random worker id
|
||||
worker_id = uuid.uuid4()
|
||||
else:
|
||||
worker_id = sys.argv[1]
|
||||
|
||||
start_packet_processing(worker_id)
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
import pandas as pd
|
||||
from kafka import KafkaConsumer
|
||||
import redis
|
||||
|
||||
from graph_based_intrusion_detection.utils import constants
|
||||
from graph_based_intrusion_detection.utils import config
|
||||
from graph_based_intrusion_detection.utils.logging import create_logger
|
||||
from graph_based_intrusion_detection.utils.state_merging import merge_states
|
||||
from graph_based_intrusion_detection.utils.network_transport_utils import deserialize, serialize
|
||||
|
||||
|
||||
def start_state_merging(worker_id: str):
|
||||
"""
|
||||
Worker function, processes states from the state queue and merges them, updating the state in the redis database
|
||||
:param worker_id:
|
||||
:return: None
|
||||
"""
|
||||
|
||||
logger = create_logger(f"State-Merger {worker_id}")
|
||||
logger.info(f"Worker started")
|
||||
|
||||
# establish connection to kafka
|
||||
state_consumer = KafkaConsumer(constants.PROCESSED_STATE_TOPIC_NAME,
|
||||
bootstrap_servers='localhost:9092',
|
||||
auto_offset_reset='earliest')
|
||||
|
||||
logger.info("Connected to kafka")
|
||||
|
||||
# establish connection to redis
|
||||
redis_host = redis.Redis(host='localhost', port=6379, db=0)
|
||||
|
||||
logger.info("Connected to redis")
|
||||
|
||||
logger.info("Initialized, waiting for states")
|
||||
|
||||
while True:
|
||||
# consume a state from kafka
|
||||
batch = list()
|
||||
while len(batch) < constants.STATE_CONSUMER_BATCH_SIZE:
|
||||
state = state_consumer.poll(timeout_ms=constants.STATE_POLL_TIMEOUT_MS,
|
||||
max_records=constants.STATE_CONSUMER_BATCH_SIZE - len(batch))
|
||||
if len(state) == 0:
|
||||
break
|
||||
|
||||
for _, records in state.items():
|
||||
for record in records:
|
||||
state = deserialize(record.value)
|
||||
batch.append(state)
|
||||
|
||||
if len(batch) == 0:
|
||||
continue
|
||||
|
||||
logger.info(f"Processing {len(batch)} state(s)")
|
||||
|
||||
# acquire lock
|
||||
lock = redis_host.lock(constants.STATE_LOCK_NAME, timeout=5)
|
||||
|
||||
# merge states in batch before updating the state in redis to keep lock time short
|
||||
batch_states_merged = dict()
|
||||
for processing_step in config.PROCESSING_STEPS:
|
||||
step_name = processing_step["name"]
|
||||
merged_step = batch[0][step_name]
|
||||
for i in range(len(batch) - 1):
|
||||
merged_step = processing_step["merge_function"](batch[i][step_name], batch[i + 1][step_name])
|
||||
|
||||
batch_states_merged[step_name] = merged_step
|
||||
|
||||
if lock.acquire(blocking=False):
|
||||
try:
|
||||
logger.debug("Acquired lock")
|
||||
# get current state from redis
|
||||
current_state_raw = redis_host.get(constants.NETWORK_STATE_KEY)
|
||||
|
||||
if current_state_raw is not None:
|
||||
current_state = deserialize(current_state_raw)
|
||||
else:
|
||||
current_state = dict()
|
||||
|
||||
# merge states
|
||||
for step in config.PROCESSING_STEPS:
|
||||
# only merge if step is present in batch, otherwise just use new state
|
||||
step_name = step["name"]
|
||||
if step_name not in current_state:
|
||||
current_state[step_name] = batch_states_merged[step_name]
|
||||
else:
|
||||
current_state[step_name] = step["merge_function"](current_state[step_name],
|
||||
batch_states_merged[step_name])
|
||||
|
||||
serialized_state = serialize(current_state)
|
||||
redis_host.set(constants.NETWORK_STATE_KEY, serialized_state)
|
||||
except Exception as e:
|
||||
logger.error(f"Error while merging states: {e}")
|
||||
finally:
|
||||
logger.debug("Releasing lock")
|
||||
lock.release()
|
||||
|
||||
logger.info(f"Merged {len(batch)} state(s), waiting for next batch")
|
||||
else:
|
||||
logger.info("Could not acquire lock, retrying in 1 second")
|
||||
continue
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
worker_id = str(uuid.uuid4())
|
||||
else:
|
||||
worker_id = sys.argv[1]
|
||||
|
||||
start_state_merging(worker_id)
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
import sys
|
||||
import uuid
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
from kafka import KafkaConsumer, KafkaProducer
|
||||
import redis
|
||||
|
||||
from graph_based_intrusion_detection.utils import constants
|
||||
from graph_based_intrusion_detection.utils.logging import create_logger
|
||||
from graph_based_intrusion_detection.utils.network_transport_utils import deserialize, serialize
|
||||
|
||||
|
||||
def start_work_dispatcher(worker_id: str):
|
||||
"""
|
||||
Worker function, creates node registry based on network state and dispatches work based on nodes that need updating
|
||||
:param worker_id: id of the worker
|
||||
:return: None
|
||||
"""
|
||||
|
||||
logger = create_logger(f"Work-Dispatcher {worker_id}")
|
||||
|
||||
# connect to kafka
|
||||
node_update_producer = KafkaProducer(
|
||||
bootstrap_servers='localhost:9092')
|
||||
|
||||
node_analysis_results_consumer = KafkaConsumer(
|
||||
constants.NODE_ANALYSIS_RESULTS_TOPIC,
|
||||
bootstrap_servers='localhost:9092',
|
||||
auto_offset_reset='earliest',
|
||||
)
|
||||
logger.info("Connected to kafka")
|
||||
|
||||
# connect to redis
|
||||
redis_host = redis.Redis(host='localhost', port=6379, db=0)
|
||||
logger.info("Connected to redis")
|
||||
|
||||
logger.info("Initialized, waiting for states")
|
||||
|
||||
awaiting_results = list()
|
||||
|
||||
# define last values for better logging
|
||||
last_network_size_key = "last_network_size"
|
||||
last_num_awaiting_results_key = "last_num_awaiting_results"
|
||||
last_values = {
|
||||
last_network_size_key: 0,
|
||||
last_num_awaiting_results_key: 0
|
||||
}
|
||||
|
||||
while True:
|
||||
# get current network state
|
||||
network_state_raw = redis_host.get(constants.NETWORK_STATE_KEY)
|
||||
|
||||
if network_state_raw is None:
|
||||
logger.info("No network state found, waiting for state")
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
network_state = deserialize(network_state_raw)
|
||||
|
||||
# get nodes from network state
|
||||
nodes = list(network_state[constants.NETWORK_STATE_CONNECTION_GRAPH_KEY].nodes())
|
||||
if len(nodes) != last_values[last_network_size_key]:
|
||||
logger.info(f"Found {len(nodes)} nodes in network state")
|
||||
last_values[last_network_size_key] = len(nodes)
|
||||
|
||||
# obtain registry lock
|
||||
lock = redis_host.lock(constants.NODE_REGISTRY_LOCK, timeout=1000)
|
||||
if lock.acquire(blocking=False):
|
||||
try:
|
||||
logger.debug("Acquired node registry lock")
|
||||
|
||||
# get current node registry
|
||||
node_registry_raw = redis_host.get(constants.NODE_REGISTRY_KEY)
|
||||
if node_registry_raw is None:
|
||||
node_registry = {}
|
||||
else:
|
||||
node_registry = deserialize(node_registry_raw)
|
||||
|
||||
# remove nodes from registry that are not in the network state
|
||||
for node in list(node_registry.keys()):
|
||||
if node not in nodes:
|
||||
del node_registry[node]
|
||||
|
||||
# get node analysis results since last update
|
||||
node_analysis_results = list()
|
||||
while True:
|
||||
node_analysis_results_raw = node_analysis_results_consumer.poll(
|
||||
timeout_ms=constants.NODE_ANALYSIS_RESULTS_POLL_TIMEOUT_MS)
|
||||
|
||||
if len(node_analysis_results_raw) == 0:
|
||||
break
|
||||
|
||||
for topic, messages in node_analysis_results_raw.items():
|
||||
for message in messages:
|
||||
result_dict = deserialize(message.value)
|
||||
node = result_dict["node"]
|
||||
node_analysis_results.append(result_dict)
|
||||
|
||||
# remove results from awaiting results
|
||||
if node in awaiting_results:
|
||||
awaiting_results.remove(node)
|
||||
|
||||
if len(awaiting_results) != last_values[last_num_awaiting_results_key]:
|
||||
logger.info(f"Found {len(awaiting_results)} nodes with awaiting results")
|
||||
last_values[last_num_awaiting_results_key] = len(awaiting_results)
|
||||
|
||||
# process node analysis results
|
||||
for analysis_result in node_analysis_results:
|
||||
node = analysis_result["node"]
|
||||
update_time = analysis_result["time"]
|
||||
results = analysis_result["results"]
|
||||
|
||||
# update node states in registry based on results
|
||||
if node in node_registry:
|
||||
node_registry[node]["last_updated"] = update_time
|
||||
node_registry[node]["results"] = results
|
||||
else:
|
||||
logger.warning(f"Received result for unknown node {node}")
|
||||
|
||||
# get nodes that are not in the registry
|
||||
new_nodes = [node for node in nodes if node not in node_registry]
|
||||
|
||||
# add new nodes to the registry
|
||||
if len(new_nodes) > 0:
|
||||
logger.info(f"Found {len(new_nodes)} new nodes in network state")
|
||||
for node in new_nodes:
|
||||
node_registry[node] = {"last_updated": None}
|
||||
|
||||
# create and dispatch work packages
|
||||
new_node_updates = list()
|
||||
for node, node_data in node_registry.items():
|
||||
last_updated = node_data["last_updated"]
|
||||
if last_updated is None or (time.time() - last_updated) > (
|
||||
constants.NODE_ANALYSIS_INTERVAL_MS / 1000):
|
||||
# only dispatch work if the node result is not already being awaited
|
||||
if node not in awaiting_results:
|
||||
node_update_message = {
|
||||
"node": node,
|
||||
"time": time.time()
|
||||
}
|
||||
node_update_producer.send(constants.NODE_ANALYSIS_TOPIC,
|
||||
value=serialize(node_update_message))
|
||||
new_node_updates.append(node)
|
||||
awaiting_results.append(node)
|
||||
|
||||
if len(new_node_updates) > 0:
|
||||
logger.info(f"Dispatched {len(new_node_updates)} new node updates")
|
||||
|
||||
# update node registry
|
||||
redis_host.set(constants.NODE_REGISTRY_KEY, serialize(node_registry))
|
||||
finally:
|
||||
# release lock
|
||||
lock.release()
|
||||
logger.debug("Released node registry lock")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
worker_id = str(uuid.uuid4())
|
||||
else:
|
||||
worker_id = sys.argv[1]
|
||||
|
||||
start_work_dispatcher(worker_id)
|
||||
@@ -0,0 +1,130 @@
|
||||
import numpy as np
|
||||
|
||||
# define processing functions
|
||||
from functools import partial
|
||||
|
||||
from graph_based_intrusion_detection.packet_processing.packet_inspection import update_port_list
|
||||
from graph_based_intrusion_detection.utils import constants
|
||||
|
||||
from graph_based_intrusion_detection.graph_processing.data_extraction import create_dataset_for_node
|
||||
|
||||
from graph_based_intrusion_detection.packet_processing.graph_processing import update_layer_3_graph
|
||||
from graph_based_intrusion_detection.utils.state_merging import merge_graph
|
||||
from graph_based_intrusion_detection.analysis.result_interpretation import interpret_prediction_result
|
||||
|
||||
PER_CONNECTION_FIELDS = [
|
||||
{
|
||||
"name": "total_packet_count",
|
||||
"min_layer": 2,
|
||||
"base_value_function": lambda _: 1,
|
||||
"update_function": lambda x, _: x + 1
|
||||
},
|
||||
{
|
||||
"name": "total_bytes",
|
||||
"min_layer": 2,
|
||||
"base_value_function": lambda packet: packet["frame/frame.len"],
|
||||
"update_function": lambda x, packet: x + packet["frame/frame.len"]
|
||||
},
|
||||
{
|
||||
"name": "src_ports",
|
||||
"min_layer": 4,
|
||||
"base_value_function": lambda packet: update_port_list(packet, [], "tcp/tcp.srcport"),
|
||||
"update_function": lambda x, packet: update_port_list(packet, x, "tcp/tcp.srcport"),
|
||||
},
|
||||
{
|
||||
"name": "dst_ports",
|
||||
"min_layer": 4,
|
||||
"base_value_function": lambda packet: update_port_list(packet, [], "tcp/tcp.dstport"),
|
||||
"update_function": lambda x, packet: update_port_list(packet, x, "tcp/tcp.dstport"),
|
||||
},
|
||||
]
|
||||
|
||||
PER_SECOND_FIELDS = [
|
||||
{
|
||||
"name": "packets_per_second",
|
||||
"min_layer": 2,
|
||||
"base_value_function": lambda _: 1,
|
||||
"update_function": lambda x, _: x + 1
|
||||
},
|
||||
{
|
||||
"name": "bytes_per_second",
|
||||
"min_layer": 2,
|
||||
"base_value_function": lambda packet: packet["frame/frame.len"],
|
||||
"update_function": lambda x, packet: x + packet["frame/frame.len"]
|
||||
},
|
||||
{
|
||||
"name": "max_tcp_segment_size",
|
||||
"min_layer": 4,
|
||||
"base_value_function": lambda packet: packet["tcp/tcp.options/tcp.options.mss/tcp.options.mss_val"],
|
||||
"update_function": lambda x, packet:
|
||||
packet["tcp/tcp.options/tcp.options.mss/tcp.options.mss_val"] if x < packet[
|
||||
"tcp/tcp.options/tcp.options.mss/tcp.options.mss_val"] else x
|
||||
},
|
||||
{
|
||||
"name": "max_tcp_header_length",
|
||||
"min_layer": 4,
|
||||
"base_value_function": lambda packet: packet["tcp/tcp.hdr_len"],
|
||||
"update_function": lambda x, packet:
|
||||
packet["tcp/tcp.hdr_len"] if x < packet["tcp/tcp.hdr_len"] else x
|
||||
},
|
||||
{
|
||||
"name": "max_frame_length",
|
||||
"min_layer": 2,
|
||||
"base_value_function": lambda packet: packet["frame/frame.len"],
|
||||
"update_function": lambda x, packet: packet["frame/frame.len"] if x < packet["frame/frame.len"] else x
|
||||
},
|
||||
{
|
||||
"name": "min_frame_length",
|
||||
"min_layer": 2,
|
||||
"base_value_function": lambda packet: packet["frame/frame.len"],
|
||||
"update_function": lambda x, packet: packet["frame/frame.len"] if x > packet["frame/frame.len"] else x
|
||||
}
|
||||
]
|
||||
|
||||
PROCESSING_STEPS = [
|
||||
{
|
||||
"name": "layer_3_graph",
|
||||
"processing_functions": [
|
||||
partial(update_layer_3_graph, per_packet_fields=PER_CONNECTION_FIELDS, per_second_fields=PER_SECOND_FIELDS,
|
||||
timedelta_field=constants.TIMEDELTA_FIELD)
|
||||
],
|
||||
"merge_function": merge_graph
|
||||
}
|
||||
]
|
||||
|
||||
# define fields for connection analysis
|
||||
NODE_ANALYSIS_PER_SECOND_ATTRIBUTES = [x["name"] for x in PER_SECOND_FIELDS]
|
||||
NODE_ANALYSIS_PER_CONNECTION_ATTRIBUTES = [x["name"] for x in PER_CONNECTION_FIELDS]
|
||||
|
||||
# define analysis model and scaler paths
|
||||
ANALYSIS_STEPS = [
|
||||
{
|
||||
"name": "incoming_attack_probability",
|
||||
"network_state_key": "layer_3_graph",
|
||||
"dataset_function": partial(create_dataset_for_node,
|
||||
per_second_attributes=NODE_ANALYSIS_PER_SECOND_ATTRIBUTES,
|
||||
per_connection_attributes=NODE_ANALYSIS_PER_CONNECTION_ATTRIBUTES),
|
||||
"dataset_identifiers": ["node"],
|
||||
"model_path": "/home/alex/projects/it-security-praktikum/devices/dev08-echo-dot-l4s3re/graph_based_intrusion_detection/resources/models/"
|
||||
"incoming_attack_clf.pickle",
|
||||
"scaler_path": "/home/alex/projects/it-security-praktikum/devices/dev08-echo-dot-l4s3re/graph_based_intrusion_detection/resources/models/"
|
||||
"layer_3_scaler.pickle",
|
||||
"predict_function": "predict_proba",
|
||||
"interpretation_function": lambda x: np.max(x[:, 1])
|
||||
},
|
||||
{
|
||||
"name": "outgoing_attack_probability",
|
||||
"network_state_key": "layer_3_graph",
|
||||
"dataset_function": partial(create_dataset_for_node,
|
||||
per_second_attributes=NODE_ANALYSIS_PER_SECOND_ATTRIBUTES,
|
||||
per_connection_attributes=NODE_ANALYSIS_PER_CONNECTION_ATTRIBUTES),
|
||||
"dataset_identifiers": ["node"],
|
||||
"model_path": "/home/alex/projects/it-security-praktikum/devices/dev08-echo-dot-l4s3re/graph_based_intrusion_detection/resources/models/"
|
||||
"outgoing_attack_nn_clf.pickle",
|
||||
"scaler_path": "/home/alex/projects/it-security-praktikum/devices/dev08-echo-dot-l4s3re/graph_based_intrusion_detection/resources/models/"
|
||||
"layer_3_scaler.pickle",
|
||||
"predict_function": "predict",
|
||||
"interpretation_function": lambda x: np.max(x[:, 1])
|
||||
}
|
||||
|
||||
]
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
import os
|
||||
from functools import partial
|
||||
|
||||
from dotenv import load_dotenv, find_dotenv
|
||||
|
||||
load_dotenv(find_dotenv())
|
||||
|
||||
from graph_based_intrusion_detection.packet_processing.graph_processing import update_layer_3_graph, \
|
||||
update_layer_2_graph
|
||||
from graph_based_intrusion_detection.packet_processing.packet_inspection import update_port_list
|
||||
|
||||
|
||||
LOG_FILE_PATH = os.getenv("LOG_FILE_PATH")
|
||||
LOG_LEVEL = os.getenv("LOG_LEVEL")
|
||||
|
||||
# packet processing
|
||||
PACKET_CONSUMER_BATCH_SIZE = 10_000
|
||||
PACKET_POLL_TIMEOUT_MS = 100
|
||||
PACKET_TOPIC_NAME = "packets"
|
||||
PROCESSED_STATE_TOPIC_NAME = "processed_states"
|
||||
|
||||
# state merging
|
||||
STATE_MERGE_TIMEOUT_MS = 100
|
||||
STATE_POLL_TIMEOUT_MS = 100
|
||||
STATE_LOCK_NAME = "network_state_lock"
|
||||
NETWORK_STATE_KEY = "network_state"
|
||||
STATE_CONSUMER_BATCH_SIZE = 10
|
||||
|
||||
# work dispatcher
|
||||
NODE_ANALYSIS_TOPIC = "node_analysis"
|
||||
NODE_ANALYSIS_RESULTS_TOPIC = "node_analysis_results"
|
||||
NODE_ANALYSIS_RESULTS_POLL_TIMEOUT_MS = 100
|
||||
NODE_ANALYSIS_INTERVAL_MS = 1000
|
||||
NETWORK_STATE_CONNECTION_GRAPH_KEY = "layer_3_graph"
|
||||
NODE_REGISTRY_KEY = "connection_registry"
|
||||
NODE_REGISTRY_LOCK = "connection_registry_lock"
|
||||
|
||||
# connection analysis
|
||||
NODE_ANALYSIS_POLL_TIMEOUT_MS = 100
|
||||
NODE_ANALYSIS_BATCH_SIZE = 100
|
||||
NODE_ANALYSIS_TIMEOUT_MS = 10_000
|
||||
SURPRESS_WARNINGS = True
|
||||
|
||||
TIME_FIELD = 'geninfo/timestamp'
|
||||
TIMEDELTA_FIELD = "timedelta"
|
||||
@@ -0,0 +1,33 @@
|
||||
from graph_based_intrusion_detection.utils.constants import LOG_FILE_PATH, LOG_LEVEL
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
def create_logger(name: str,
|
||||
log_file: str = LOG_FILE_PATH,
|
||||
log_level: str = LOG_LEVEL,
|
||||
) -> logging.Logger:
|
||||
"""
|
||||
Create a logger for a thread
|
||||
:param name: name of the logger
|
||||
:param log_file: path to the log file
|
||||
:param log_level: log level
|
||||
:return: logger
|
||||
"""
|
||||
logger = logging.getLogger(name)
|
||||
logger.setLevel(eval(f"logging.{log_level}"))
|
||||
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
|
||||
# add stream handler
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging.DEBUG)
|
||||
logger.addHandler(ch)
|
||||
ch.setFormatter(formatter)
|
||||
|
||||
# add file handler
|
||||
fh = logging.FileHandler(log_file)
|
||||
fh.setLevel(logging.DEBUG)
|
||||
fh.setFormatter(formatter)
|
||||
logger.addHandler(fh)
|
||||
|
||||
return logger
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
from datetime import datetime, timedelta
|
||||
import json
|
||||
import pickle
|
||||
|
||||
import networkx as nx
|
||||
from networkx.readwrite import json_graph
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def serialize_series(series: pd.Series) -> bytes:
|
||||
"""
|
||||
Serialize a pandas series to a dictionary containing the data and the dtype
|
||||
:param series: series to serialize
|
||||
:return: dictionary containing the data and the dtype
|
||||
"""
|
||||
|
||||
return series.to_parquet()
|
||||
|
||||
|
||||
def deserialize_series(series_dict: dict) -> pd.Series:
|
||||
"""
|
||||
Deserialize a dictionary to a pandas series
|
||||
:param series_dict: dictionary containing the data and the dtype
|
||||
:return: pandas series
|
||||
"""
|
||||
|
||||
return pd.Series(series_dict["data"]).astype(series_dict["dtype"])
|
||||
|
||||
|
||||
def serialize_graph(graph: nx.Graph) -> dict:
|
||||
"""
|
||||
Serialize a networkx graph to a dictionary containing the data and the dtype
|
||||
:param graph: graph to serialize
|
||||
:return: dictionary containing the data and the dtype
|
||||
"""
|
||||
|
||||
return json_graph.node_link_data(graph)
|
||||
|
||||
|
||||
def deserialize_graph(graph_dict: dict) -> nx.Graph:
|
||||
"""
|
||||
Deserialize a dictionary to a networkx graph
|
||||
:param graph_dict: dictionary containing the data and the dtype
|
||||
:return: networkx graph
|
||||
"""
|
||||
|
||||
return json_graph.node_link_graph(graph_dict)
|
||||
|
||||
|
||||
def serialize(value) -> bytes:
|
||||
"""
|
||||
Serialize a value by using pickle
|
||||
:param value: value to serialize
|
||||
:return: serialized value as bytes
|
||||
"""
|
||||
|
||||
return pickle.dumps(value)
|
||||
|
||||
|
||||
def deserialize(value: bytes):
|
||||
"""
|
||||
Deserialize a value by using pickle
|
||||
:param value: value to deserialize
|
||||
:return: deserialized value
|
||||
"""
|
||||
return pickle.loads(value)
|
||||
|
||||
# def serialize(value):
|
||||
# """
|
||||
# Serialize a value
|
||||
# :param value: any value
|
||||
# :return: serialized value
|
||||
# """
|
||||
# if isinstance(value, nx.Graph):
|
||||
# return serialize_graph(value)
|
||||
# elif isinstance(value, dict):
|
||||
# return {k: serialize(v) for k, v in value.items()}
|
||||
# elif isinstance(value, list) or isinstance(value, tuple):
|
||||
# return [serialize(v) for v in value]
|
||||
# elif isinstance(value, pd.DataFrame):
|
||||
# return serialize(value.to_dict())
|
||||
# elif isinstance(value, datetime):
|
||||
# return value.isoformat()
|
||||
# elif isinstance(value, timedelta):
|
||||
# return value.total_seconds()
|
||||
# else:
|
||||
# return value
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
import copy
|
||||
from typing import Any
|
||||
|
||||
import networkx as nx
|
||||
|
||||
|
||||
def merge_states(state_set: list) -> dict:
|
||||
"""
|
||||
Merge states
|
||||
:param state_set: list of states
|
||||
:return:
|
||||
"""
|
||||
|
||||
if len(state_set) == 0:
|
||||
raise ValueError("No states to merge")
|
||||
|
||||
if len(state_set) == 1:
|
||||
return state_set[0]
|
||||
|
||||
merged_state = state_set[0]
|
||||
for state in state_set[1:]:
|
||||
for key, value in state.items():
|
||||
if key in merged_state:
|
||||
if isinstance(merged_state[key], nx.Graph):
|
||||
merged_state[key] = merge_graph(merged_state[key], value)
|
||||
elif isinstance(merged_state[key], dict):
|
||||
for sub_key, sub_value in value.items():
|
||||
merge_by_key(merged_state[key], sub_key, sub_value)
|
||||
elif isinstance(merged_state[key], list):
|
||||
merged_state[key] = merged_state[key] + [x for x in value if x not in merged_state[key]]
|
||||
elif merged_state[key] == value:
|
||||
pass
|
||||
else:
|
||||
merged_state[key] = merged_state[key] + value
|
||||
else:
|
||||
merged_state[key] = value
|
||||
|
||||
return merged_state
|
||||
|
||||
|
||||
def merge_by_key(base_dict: dict, key: str | int, value: Any):
|
||||
"""
|
||||
Merge a value into a dictionary by key. If the key is not present, it is added, otherwise the value is updated by addition
|
||||
:param base_dict: base dictionary to update
|
||||
:param key: key to update
|
||||
:param value: value to update with
|
||||
:return: nothing, base_dict is updated in place
|
||||
"""
|
||||
if key not in base_dict:
|
||||
base_dict[key] = value
|
||||
else:
|
||||
if isinstance(base_dict[key], dict):
|
||||
for sub_key, sub_value in value.items():
|
||||
merge_by_key(base_dict[key], sub_key, sub_value)
|
||||
elif isinstance(base_dict[key], list):
|
||||
if isinstance(value, list):
|
||||
base_dict[key] = base_dict[key] + [x for x in value if x not in base_dict[key]]
|
||||
else:
|
||||
if value not in base_dict[key]:
|
||||
base_dict[key].append(value)
|
||||
else:
|
||||
if key in base_dict:
|
||||
try:
|
||||
base_dict[key] = base_dict[key] + value
|
||||
except:
|
||||
pass
|
||||
else:
|
||||
base_dict[key] = value
|
||||
|
||||
|
||||
def merge_graph(g: nx.Graph, h: nx.Graph) -> nx.Graph:
|
||||
"""
|
||||
Merge two graphs, combines the weights of the edges based on the keys and tries to add their values
|
||||
:param g: first graph
|
||||
:param h: second graph
|
||||
:return: merged graph
|
||||
"""
|
||||
|
||||
if g is None:
|
||||
return h
|
||||
|
||||
if h is None:
|
||||
return g
|
||||
|
||||
out = copy.deepcopy(g)
|
||||
|
||||
# iterate over nodes in h
|
||||
for node in h.nodes():
|
||||
if node not in out.nodes:
|
||||
out.add_node(node)
|
||||
|
||||
# iterate over edges in h
|
||||
for src, dest, data in h.edges(data=True):
|
||||
if out.has_edge(src, dest):
|
||||
for key, value in data.items():
|
||||
merge_by_key(out[src][dest], key, value)
|
||||
else:
|
||||
out.add_edge(src, dest, **data)
|
||||
|
||||
return out
|
||||
Reference in New Issue
Block a user