alex/annescribe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first 'working' version

Browse Source
This commit is contained in:
Alex
2024-12-29 23:43:26 +01:00
parent 6200763f0b
commit b3c30398c2
11 changed files with 469 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+309
View File
@@ -0,0 +1,309 @@
auth.yml
uploads
jobs
.idea
# Created by https://www.toptal.com/developers/gitignore/api/pycharm,python,venv
# Edit at https://www.toptal.com/developers/gitignore?templates=pycharm,python,venv
### PyCharm ###
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
# AWS User-specific
.idea/**/aws.xml
# Generated files
.idea/**/contentModel.xml
# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
# Gradle
.idea/**/gradle.xml
.idea/**/libraries
# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using
# auto-import.
# .idea/artifacts
# .idea/compiler.xml
# .idea/jarRepositories.xml
# .idea/modules.xml
# .idea/*.iml
# .idea/modules
# *.iml
# *.ipr
# CMake
cmake-build-*/
# Mongo Explorer plugin
.idea/**/mongoSettings.xml
# File-based project format
*.iws
# IntelliJ
out/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Cursive Clojure plugin
.idea/replstate.xml
# SonarLint plugin
.idea/sonarlint/
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
# Editor-based Rest Client
.idea/httpRequests
# Android studio 3.1+ serialized cache file
.idea/caches/build_file_checksums.ser
### PyCharm Patch ###
# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
# *.iml
# modules.xml
# .idea/misc.xml
# *.ipr
# Sonarlint plugin
# https://plugins.jetbrains.com/plugin/7973-sonarlint
.idea/**/sonarlint/
# SonarQube Plugin
# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
.idea/**/sonarIssues.xml
# Markdown Navigator plugin
# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
.idea/**/markdown-navigator.xml
.idea/**/markdown-navigator-enh.xml
.idea/**/markdown-navigator/
# Cache file creation bug
# See https://youtrack.jetbrains.com/issue/JBR-2257
.idea/$CACHE_FILE$
# CodeStream plugin
# https://plugins.jetbrains.com/plugin/12206-codestream
.idea/codestream.xml
# Azure Toolkit for IntelliJ plugin
# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij
.idea/**/azureSettings.xml
### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
### Python Patch ###
# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
poetry.toml
# ruff
.ruff_cache/
# LSP config files
pyrightconfig.json
### venv ###
# Virtualenv
# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
[Bb]in
[Ii]nclude
[Ll]ib
[Ll]ib64
[Ll]ocal
[Ss]cripts
pyvenv.cfg
pip-selfcheck.json
.python-version
./venv
# End of https://www.toptal.com/developers/gitignore/api/pycharm,python,venv
.streamlit/config.toml
+2
View File
@@ -0,0 +1,2 @@
[client]
toolbarMode = "viewer"
config.yml
+15
View File
@@ -0,0 +1,15 @@
jobs:
root_directory: ./jobs
data:
upload_directory: ./uploads
audio:
chunking:
min_silence_level: -40
min_silence_length: 200
ms_silence_to_keep: 100
model:
type: base
device: cuda
requirements.txt
+5
View File
@@ -0,0 +1,5 @@
streamlit
streamlit-authenticator
streamlit-autorefresh
openai-whisper
pydub
transcription/job_utils.py
+3
View File
@@ -190,6 +190,9 @@ def get_progress(job_name: str, config: dict) -> int:
input_dir = os.path.join(config['jobs']['root_directory'], job_name, chunk_input_subdir)
output_dir = os.path.join(config['jobs']['root_directory'], job_name, chunk_output_subdir)
if not os.path.exists(input_dir) or not os.path.exists(output_dir):
return 0
num_input_files = len(os.listdir(input_dir))
num_output_files = len(os.listdir(output_dir))
transcription/transcription_helper.py
+6 -6
View File
@@ -3,11 +3,11 @@ import os
from multiprocessing import Lock
from concurrent.futures import ProcessPoolExecutor
import job_utils
from utils import get_logger
from audio_processing import split_audio, convert_to_wav
from job_utils import is_job, remove_job, create_job, get_processing, set_processing
from transcription import transcribe_audio
from transcription import job_utils
from transcription.job_utils import is_job, remove_job, create_job, get_processing, set_processing
from transcription.audio_processing import split_audio, convert_to_wav
from transcription.transcription import transcribe_audio
job_update_lock = Lock()
default_input_audio_file_name = 'input_audio'
@@ -36,7 +36,7 @@ def transcribe(audio_file_path: str,
job_name = os.path.basename(audio_file_path).split('.')[0]
job_dir = os.path.join(config['jobs']['root_directory'], job_name)
logger.info(f"Transcribing {audio_file_path} to {job_dir}")
logger.info(f"Transcribing {audio_file_path} to {job_dir}, blocking={blocking}, overwrite={overwrite}")
# check, if a job with the same audio file already exists
job_already_exists = is_job(job_name, config)
@@ -129,7 +129,7 @@ def run_transcription_job(job_name: str,
logger.info(f'Combining output text chunks for job {job_name}')
output_chunk_folder = os.path.join(config['jobs']['root_directory'], job_name, job_utils.chunk_output_subdir)
output_text = ''
for text_chunk_file in sorted(os.listdir(output_chunk_folder)):
for text_chunk_file in sorted(os.listdir(output_chunk_folder), key=lambda x: int(x.split('_')[1].split('.')[0])):
with open(os.path.join(output_chunk_folder, text_chunk_file), 'r') as text_chunk:
output_text += text_chunk.read() + ' '
transcription/utils.py → utils.py
View File
webapp/home.py
+10 -2
View File
@@ -1,5 +1,8 @@
from functools import partial
import streamlit as st
from webapp.new_task import render_new_task
from webapp.task_list import render_task_list
@@ -16,7 +19,12 @@ def render_home(container,
with container.container():
st.title('Annescribe Home')
st.write("Welcome to AnneScribe!")
authenticator.logout()
authenticator.logout(callback=st.session_state.clear)
# render task list
render_task_list(container, app_config)
task_list_container = st.empty()
render_task_list(task_list_container, app_config)
# render new task
new_task_container = st.empty()
render_new_task(new_task_container, app_config)
webapp/new_task.py
+61
View File
@@ -0,0 +1,61 @@
import os
import streamlit as st
from utils import get_logger
from transcription.job_utils import create_job
from transcription.transcription_helper import transcribe
logger = get_logger(__name__)
def create_new_task(filepath: str, app_config, overwrite: bool):
"""
Create a new task
:param filepath: path to the audio file
:param app_config: app configuration
:param overwrite: overwrite existing task
:return: None
"""
logger.info(f"Creating new task for file {filepath}")
if filepath is None:
st.error("No file uploaded")
return
transcribe(filepath, app_config, blocking=False, overwrite=overwrite)
def render_new_task(container, app_config: dict):
"""
Render the new task page
:param container: container object to place the content in
:param app_config: app configuration
:return: None
"""
with container.container():
st.write("Create a new task")
# add checkbox for overwrite
with st.form(key="new_task_form", clear_on_submit=True):
overwrite = st.checkbox("Overwrite existing task", value=False, key="overwrite")
uploaded_file = st.file_uploader("Audio File", type=["wav", "mp3", "flac", "ogg"], key="audio_file")
file_path = None
if uploaded_file is not None:
# save files to upload directory
upload_directory = app_config["data"]["upload_directory"]
# create the upload directory if it does not exist
if not os.path.exists(upload_directory):
logger.info(f"Creating upload directory {upload_directory}")
os.makedirs(upload_directory)
logger.info(f"Saving file {uploaded_file.name} to {upload_directory}")
file_path = f"{upload_directory}/{uploaded_file.name}"
with open(file_path, "wb") as file:
file.write(uploaded_file.getbuffer())
st.success(f"File saved to {file_path}")
st.form_submit_button("Create Task", on_click=create_new_task, args=(file_path, app_config, overwrite))
webapp/task_list.py
+49 -19
View File
@@ -4,7 +4,8 @@ import streamlit as st
from streamlit_autorefresh import st_autorefresh
from transcription.job_utils import get_existing_jobs
from transcription.job_utils import get_existing_jobs, remove_job
def show_results(job_data: dict):
"""
@@ -15,8 +16,11 @@ def show_results(job_data: dict):
if job_data is None:
return
else:
st.session_state[f"results_button_{job_data['name']}"] = True
st.write(f"Transcription Results for job {job_data['name']}")
# print("showing results")
# st.session_state[f"shown_result"] = job_data['name']
# st.write(f"Transcription Results for job {job_data['name']}")
# st.write(f"Output: {job_data['output']}")
with st.popover("Transcription Results", f"Results for job {job_data['name']}"):
st.write(f"Output: {job_data['output']}")
def hide_results(job_data: dict):
@@ -25,7 +29,7 @@ def hide_results(job_data: dict):
:param job_data: job data
:return: None
"""
st.session_state[f"results_button_{job_data['name']}"] = False
st.session_state[f"shown_result"] = None
def results_button(job_data: dict, app_config: dict):
"""
@@ -34,16 +38,33 @@ def results_button(job_data: dict, app_config: dict):
:param app_config: app configuration
:return: None
"""
is_toggled = st.session_state.get(f"results_button_{job_data['name']}")
other_toggled = any([st.session_state.get(f"results_button_{job['name']}") for job in get_existing_jobs(app_config)])
# is_toggled = st.session_state.get(f"shown_result") == job_data['name']
#
# if is_toggled:
# return st.button("Hide Results", on_click=partial(hide_results, job_data), key=f"hide_{job_data['name']}")
# else:
# return st.button("Show Results", on_click=partial(show_results, job_data), key=f"show_{job_data['name']}")
with st.popover("Show Results"):
st.markdown("Results")
st.write(job_data['output'])
@st.dialog("Are you sure?")
def delete_task(job_data: dict, app_config: dict):
"""
Delete a task
:param job_data: job data
:param app_config: app configuration
:return: None
"""
st.write(f"Are you sure you want to delete task {job_data['name']}?")
if st.button("Yes"):
remove_job(job_data['name'], app_config)
st.success(f"Task {job_data['name']} deleted.")
st.rerun()
if other_toggled:
for job in get_existing_jobs(app_config):
st.session_state[f"results_button_{job['name']}"] = False
if is_toggled:
return st.button("Show Results", on_click=partial(show_results, job_data))
else:
return st.button("Hide Results", on_click=partial(hide_results, job_data))
def render_task_list(container, app_config: dict):
@@ -54,15 +75,11 @@ def render_task_list(container, app_config: dict):
:return: None
"""
# disable all results
for job in get_existing_jobs(app_config):
st.session_state[f"results_button_{job['name']}"] = False
existing_jobs = get_existing_jobs(app_config)
with container.container():
st.subheader('Task List')
existing_jobs = get_existing_jobs(app_config)
column_config = [
{
"name": "Name",
@@ -79,19 +96,32 @@ def render_task_list(container, app_config: dict):
{
"name": "Output",
"render_func": lambda x: results_button(x, app_config)
},
{
"name": "Actions",
"render_func": lambda x: st.button("Delete", on_click=partial(delete_task, x, app_config), key=f"delete_{x['name']}")
}
]
if len(existing_jobs) == 0:
st.write("No tasks available.")
else:
# display header
cols = st.columns(len(column_config))
for i, col in enumerate(column_config):
with cols[i]:
st.write(col["name"])
for i, job in enumerate(existing_jobs):
cols = st.columns(len(column_config))
for j, col in enumerate(column_config):
cols[j].write(col["name"])
with cols[j]:
col["render_func"](job)
def update_job_list():
nonlocal existing_jobs
existing_jobs = get_existing_jobs(app_config)
st.button("Refresh", on_click=update_job_list)
webapp_main.py
+6 -6
View File
@@ -43,16 +43,16 @@ def render_login():
with content_wrapper.container():
st.title('Annescribe Login')
st.write("Please login with your credentials.")
authenticator.login(callback=st.session_state.clear())
authenticator.login()
if "authentication_status" not in st.session_state.keys():
authentication_status = None
st.session_state["authentication_status"] = authentication_status
else:
if "authentication_status" not in st.session_state:
print("schniedel")
st.session_state["authentication_status"] = None
st.session_state["username"] = None
authentication_status = st.session_state.get("authentication_status")
username = st.session_state.get("username")
print(authentication_status)
if authentication_status is True:
render_home(content_wrapper, config, authenticator)
elif authentication_status is None: