From dcaabd18e9a5be9ab8b589882cd698206d904aa8 Mon Sep 17 00:00:00 2001 From: Alex Blank Date: Tue, 31 Dec 2024 14:59:35 +0100 Subject: [PATCH] further work and redesign --- .gitignore | 1 + config.yml | 15 ------ src/annescribe/transcription/job_utils.py | 64 +++++++++++++++++++++++ src/annescribe/utils.py | 6 +++ src/annescribe/webapp/home.py | 16 +++--- src/annescribe/webapp/new_task.py | 11 ++-- src/annescribe/webapp/task_list.py | 62 +++++++++++++++++++--- src/annescribe/webapp_main.py | 13 +++-- 8 files changed, 152 insertions(+), 36 deletions(-) delete mode 100644 config.yml diff --git a/.gitignore b/.gitignore index 6562e5c..bef298f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ auth.yml +config.yml uploads jobs diff --git a/config.yml b/config.yml deleted file mode 100644 index 58fb399..0000000 --- a/config.yml +++ /dev/null @@ -1,15 +0,0 @@ -jobs: - root_directory: ./jobs - -data: - upload_directory: ./uploads - -audio: - chunking: - min_silence_level: -40 - min_silence_length: 200 - ms_silence_to_keep: 100 - -model: - type: base - device: cpu \ No newline at end of file diff --git a/src/annescribe/transcription/job_utils.py b/src/annescribe/transcription/job_utils.py index 9dd856e..db682ab 100644 --- a/src/annescribe/transcription/job_utils.py +++ b/src/annescribe/transcription/job_utils.py @@ -1,5 +1,6 @@ import os import shutil +from datetime import datetime from multiprocessing import Lock # multiprocessing lock for thread safety @@ -203,3 +204,66 @@ def get_progress(job_name: str, config: dict) -> int: return 100 return int(num_output_files / num_input_files * 100) + + +def get_progress_details(job_name: str, config: dict) -> dict: + """ + Get the progress details of a job. + :param job_name: name of the job + :param config: app configuration + :return: progress details of the job + """ + + job_dir = os.path.join(config['jobs']['root_directory'], job_name) + input_dir = os.path.join(config['jobs']['root_directory'], job_name, chunk_input_subdir) + output_dir = os.path.join(config['jobs']['root_directory'], job_name, chunk_output_subdir) + + start_time = datetime.fromtimestamp(os.path.getctime(job_dir)) + start_time_formatted = start_time.strftime("%d.%m.%Y %H:%M:%S") + + if not os.path.exists(input_dir): + return { + "progress_details": "Preparing chunking of audio file", + "time_remaining": None, + "start_time": start_time_formatted, + } + + num_input_files = len(os.listdir(input_dir)) + if not os.path.exists(output_dir): + return { + "progress_details": "Chunking audio file, current number of chunks: " + str(num_input_files), + "time_remaining": None, + "start_time": start_time_formatted, + } + + # get creation time of newest output file + num_output_files = len(os.listdir(output_dir)) + if num_output_files == 0: + return { + "progress_details": "Preparing processing of chunks", + "time_remaining": None, + "start_time": start_time_formatted, + } + + newest_output_file = max([os.path.getctime(os.path.join(output_dir, f)) for f in os.listdir(output_dir)]) + latest_chunk_time = datetime.fromtimestamp(newest_output_file) + + + # calculate time remaining + time_remaining = (latest_chunk_time - start_time) / num_output_files * (num_input_files - num_output_files) + + # format time remaining + time_remaining_formatted = "" + if time_remaining.days > 0: + time_remaining_formatted += f"{time_remaining.days} days, " + if time_remaining.seconds // 3600 > 0: + time_remaining_formatted += f"{time_remaining.seconds // 3600} hours, " + if (time_remaining.seconds // 60) % 60 > 0: + time_remaining_formatted += f"{(time_remaining.seconds // 60) % 60} minutes, " + time_remaining_formatted += f"{time_remaining.seconds % 60} seconds" + + return { + "progress_details": f"Processing chunk {num_output_files} of {num_input_files}", + "time_remaining": time_remaining_formatted, + "start_time": start_time_formatted, + } diff --git a/src/annescribe/utils.py b/src/annescribe/utils.py index 13a1dab..b778128 100644 --- a/src/annescribe/utils.py +++ b/src/annescribe/utils.py @@ -9,7 +9,13 @@ def get_logger(name: str) -> Logger: :param name: name of the logger :return: logger instance """ + logger = logging.getLogger(name) + + # check if logger has already been configured + if logger.hasHandlers(): + return logger + logger.setLevel(logging.INFO) # create a file handler diff --git a/src/annescribe/webapp/home.py b/src/annescribe/webapp/home.py index 09d50d8..1e34790 100644 --- a/src/annescribe/webapp/home.py +++ b/src/annescribe/webapp/home.py @@ -17,14 +17,18 @@ def render_home(container, """ with container.container(): - st.title('Annescribe Home') - st.write("Welcome to AnneScribe!") - authenticator.logout(callback=st.session_state.clear) + col1, col2 = st.columns([3, 1]) + with col1: + st.title('Annescribe Home') + st.write("Welcome to AnneScribe, the audio transcription tool for your every Masterarbeit need.") + with col2: + authenticator.logout(callback=lambda x: st.session_state.clear()) + + # spacer + st.divider() # render task list task_list_container = st.empty() render_task_list(task_list_container, app_config) - # render new task - new_task_container = st.empty() - render_new_task(new_task_container, app_config) + diff --git a/src/annescribe/webapp/new_task.py b/src/annescribe/webapp/new_task.py index a173cdd..4e6fb3a 100644 --- a/src/annescribe/webapp/new_task.py +++ b/src/annescribe/webapp/new_task.py @@ -58,11 +58,16 @@ def render_new_task(container, app_config: dict): @st.dialog("Upload new audio file") def show_create_task_dialog(): - st.write("Create a transcription job") + st.write("Create a transcription job for your audio file") + st.write("Please upload an audio file in WAV, MP3, FLAC, or OGG format and create a new task.") + st.write("The task will be processed in the background and will show up in the task list.") # add checkbox for overwrite - overwrite = st.checkbox("Overwrite existing task", value=False, key="overwrite") + overwrite = st.checkbox("Enable this to rerun the transcription for an audio file with the same name", value=False, key="overwrite") uploaded_file = st.file_uploader("Audio File", type=["wav", "mp3", "flac", "ogg"]) - if st.button("Create Task", key="btn_create_task"): + if st.button("Create Task", key="btn_create_task", disabled=uploaded_file is None): + if uploaded_file is None: + st.error("No file uploaded") + return create_new_task(uploaded_file, app_config, overwrite=overwrite) st.write("Task created") st.rerun() diff --git a/src/annescribe/webapp/task_list.py b/src/annescribe/webapp/task_list.py index df73e18..cf02969 100644 --- a/src/annescribe/webapp/task_list.py +++ b/src/annescribe/webapp/task_list.py @@ -3,8 +3,8 @@ from functools import partial import streamlit as st from streamlit_autorefresh import st_autorefresh - -from annescribe.transcription.job_utils import get_existing_jobs, remove_job +from annescribe.transcription.job_utils import get_existing_jobs, remove_job, get_progress_details +from annescribe.webapp.new_task import render_new_task def show_results(job_data: dict): @@ -23,6 +23,7 @@ def show_results(job_data: dict): with st.popover("Transcription Results", f"Results for job {job_data['name']}"): st.write(f"Output: {job_data['output']}") + def hide_results(job_data: dict): """ Hide the results of a job @@ -31,6 +32,7 @@ def hide_results(job_data: dict): """ st.session_state[f"shown_result"] = None + def results_button(job_data: dict, app_config: dict): """ Show the results of a job @@ -49,6 +51,7 @@ def results_button(job_data: dict, app_config: dict): st.markdown("Results") st.write(job_data['output']) + @st.dialog("Are you sure?") def delete_task(job_data: dict, app_config: dict): """ @@ -64,7 +67,44 @@ def delete_task(job_data: dict, app_config: dict): st.rerun() +def render_progress(job_data: dict): + """ + Render the progress of a job + :param job_data: job data + :return: None + """ + if job_data["completed"]: + st.markdown("--") + return + + st.progress(job_data["progress"]) + + +def render_status(job_data: dict, app_config: dict): + """ + Render the status of a job + :param job_data: job data + :param app_config: app configuration + :return: None + """ + if job_data["completed"]: + st.markdown("Completed") + elif job_data["processing"]: + # get processing details + processing_details = get_progress_details(job_data["name"], app_config) + details = processing_details["progress_details"] + start_time = processing_details["start_time"] + time_remaining = processing_details["time_remaining"] + + cols = st.columns(3) + st.markdown(f"{details}") + st.markdown(f"Started: {start_time if start_time is not None else 'N/A'}") + st.markdown(f"Time remaining: {time_remaining if time_remaining is not None else 'N/A'}") + + + else: + st.markdown("Pending") def render_task_list(container, app_config: dict): @@ -79,6 +119,7 @@ def render_task_list(container, app_config: dict): with container.container(): st.subheader('Task List') + st.write("Here you can see the status of your tasks, view the results, and delete tasks.") column_config = [ { @@ -87,11 +128,11 @@ def render_task_list(container, app_config: dict): }, { "name": "Progress", - "render_func": lambda x: st.progress(x["progress"]) if x["progress"] < 100 else st.markdown(" -- ") + "render_func": lambda x: render_progress(x) }, { "name": "Status", - "render_func": lambda x: st.markdown("Completed") if x["completed"] else st.markdown("Processing" if x["processing"] else "Pending") + "render_func": lambda x: render_status(x, app_config) }, { "name": "Output", @@ -99,7 +140,8 @@ def render_task_list(container, app_config: dict): }, { "name": "Actions", - "render_func": lambda x: st.button("Delete", on_click=partial(delete_task, x, app_config), key=f"delete_{x['name']}") + "render_func": lambda x: st.button("Delete", on_click=partial(delete_task, x, app_config), + key=f"delete_{x['name']}") } ] @@ -116,12 +158,18 @@ def render_task_list(container, app_config: dict): for j, col in enumerate(column_config): with cols[j]: col["render_func"](job) + st.divider() def update_job_list(): nonlocal existing_jobs existing_jobs = get_existing_jobs(app_config) - st.button("Refresh", on_click=update_job_list) - + st.write( + "Unfortunately, the list does not update automatically, so you have to hit refresh manually every once in a while.") + col1, col2, col3 = st.columns([.4, 1, 5]) + with col1: + st.button("Refresh", on_click=update_job_list) + with col2: + render_new_task(col2, app_config) diff --git a/src/annescribe/webapp_main.py b/src/annescribe/webapp_main.py index 122d948..dd2d04c 100644 --- a/src/annescribe/webapp_main.py +++ b/src/annescribe/webapp_main.py @@ -36,6 +36,13 @@ else: logger.info(f"AUTH_FILE {AUTH_FILE} does not exist, user authentication disabled") authentication_enabled = False +# set wide mode +st.set_page_config( + page_title="Annescribe", + layout="wide", + initial_sidebar_state="auto", +) + if authentication_enabled: authenticator = st_auth.Authenticate( auth_config["credentials"], @@ -46,11 +53,9 @@ if authentication_enabled: content_wrapper = st.empty() - def render_login(): with content_wrapper.container(): st.title('Annescribe Login') - st.write("Please login with your credentials.") if authentication_enabled: authenticator.login() @@ -64,11 +69,9 @@ authentication_status = st.session_state.get("authentication_status") username = st.session_state.get("username") if authentication_status is True: render_home(content_wrapper, config, authenticator if authentication_enabled else None) -# elif authentication_status is None: -# st.warning('Please enter your username and password') -# render_login() else: # make sure to remove cookie + print("rendering login...") render_login()