This commit is contained in:
2026-05-13 09:37:42 -04:00
parent ca3469376a
commit 2889993a7d
40 changed files with 1583 additions and 0 deletions

10
twitch-vod/.env.example Normal file
View File

@@ -0,0 +1,10 @@
# Twitch API credentials
TWITCH_CLIENT_ID=your-twitch-client-id
TWITCH_ACCESS_TOKEN=your-twitch-access-token
# Postgres configuration
POSTGRES_HOST=localhost
POSTGRES_PORT=5432
POSTGRES_DB=twitch_vod
POSTGRES_USER=postgres
POSTGRES_PASSWORD=postgres

146
twitch-vod/.gitignore vendored Normal file
View File

@@ -0,0 +1,146 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
Pipfile.lock
# poetry
poetry.lock
# pdm
.pdm.toml
# PEP 582
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# IDEs
.vscode/
.idea/
*.swp
*.swo
*~
# Project specific
vods/

8
twitch-vod/Makefile Normal file
View File

@@ -0,0 +1,8 @@
.PHONY: tui
tui:
python ./src/twitch_vod/tui.py
.PHONY: run
run:
fastapi dev ./src/twitch_vod/main.py

0
twitch-vod/README.md Normal file
View File

View File

@@ -0,0 +1,47 @@
services:
postgres:
image: postgres:16-alpine
container_name: twitch-vod-postgres
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: twitch_vod
ports:
- "5432:5432"
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 10s
timeout: 5s
retries: 5
pgadmin:
image: dpage/pgadmin4:latest
container_name: twitch-vod-pgadmin
environment:
PGADMIN_DEFAULT_EMAIL: admin@admin.com
PGADMIN_DEFAULT_PASSWORD: admin
PGADMIN_LISTEN_PORT: 80
ports:
- "5050:80"
depends_on:
postgres:
condition: service_healthy
minio:
image: minio/minio:latest
container_name: twitch-vod-minio
environment:
MINIO_ROOT_USER: minioadmin
MINIO_ROOT_PASSWORD: minioadmin
ports:
- "9000:9000"
- "9001:9001"
volumes:
- minio_data:/data
command: server /data --console-address ":9001"
volumes:
postgres_data:
minio_data:

29
twitch-vod/pyproject.toml Normal file
View File

@@ -0,0 +1,29 @@
[project]
name = "twitch-vod"
version = "0.1.0"
description = ""
authors = [
{name = "Collin Campbell",email = "collincampbell97@proton.me"}
]
readme = "README.md"
requires-python = ">=3.10,<4"
dependencies = [
"streamlink (>=8.1.2,<9.0.0)",
"twitchapi (>=4.5.0,<5.0.0)",
"dotenv (>=0.9.9,<0.10.0)",
"loguru (>=0.7.3,<0.8.0)",
"pydantic (>=2.12.5,<3.0.0)",
"textual (>=7.5.0,<8.0.0)",
"textual-dev (>=1.8.0,<2.0.0)",
"psycopg2-binary (>=2.9.9,<3.0.0)",
"boto3 (>=1.35.0,<2.0.0)",
"fastapi[standard] (>=0.128.5,<0.129.0)",
]
[tool.poetry]
packages = [{include = "twitch_vod", from = "src"}]
[build-system]
requires = ["poetry-core>=2.0.0,<3.0.0"]
build-backend = "poetry.core.masonry.api"

View File

View File

@@ -0,0 +1,61 @@
import os
from dotenv import load_dotenv
load_dotenv()
basedir = os.path.abspath(os.path.dirname(__file__))
class Config(object):
TWITCH_CLIENT_ID = os.getenv("TWITCH_CLIENT_ID")
TWITCH_SECRET = os.getenv("TWITCH_SECRET")
POSTGRES_HOST = os.getenv("POSTGRES_HOST")
POSTGRES_PORT = os.getenv("POSTGRES_PORT")
POSTGRES_DB = os.getenv("POSTGRES_DB")
POSTGRES_USER = os.getenv("POSTGRES_USER")
POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD")
S3_ENDPOINT_URL = os.getenv("S3_ENDPOINT_URL")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY")
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY")
S3_REGION = os.getenv("S3_REGION", "us-east-1")
def __init__(self):
required_vars = [
"TWITCH_CLIENT_ID",
"TWITCH_SECRET",
"POSTGRES_HOST",
"POSTGRES_PORT",
"POSTGRES_DB",
"POSTGRES_USER",
"POSTGRES_PASSWORD",
"S3_ENDPOINT_URL",
"S3_ACCESS_KEY",
"S3_SECRET_KEY",
]
missing = [var for var in required_vars if not os.getenv(var)]
if missing:
raise ValueError(
f"Missing required environment variables: {', '.join(missing)}"
)
class DevelopmentConfig(Config):
DEBUG = True
LOG_BACKTRACE = True
LOG_LEVEL = "DEBUG"
class ProductionConfig(Config):
LOG_BACKTRACE = False
LOG_LEVEL = "INFO"
config_types = {
"development": DevelopmentConfig,
"production": ProductionConfig,
"default": DevelopmentConfig,
}

View File

@@ -0,0 +1,37 @@
import logging
# need to import the models to allow SQLModel to create the DB tables for us
from twitch_vod.services.db.models import * # noqa # pylint: disable=unused-import
from contextlib import asynccontextmanager
from fastapi import FastAPI
from sqlmodel import SQLModel
from twitch_vod.services.db.postgres import postgres_engine
from twitch_vod.routers import channel
from twitch_vod.services.twitch.controller import TwitchController
logger = logging.getLogger(__name__)
@asynccontextmanager
async def lifespan(app: FastAPI):
logger.info("application starting")
engine = postgres_engine()
twitch_controller = await TwitchController().create()
SQLModel.metadata.create_all(engine)
yield
logger.info("application is shutting down")
engine.dispose()
twitch_controller.cleanup()
app = FastAPI(lifespan=lifespan)
app.include_router(channel.router)
@app.get("/")
async def root():
return {"healthy"}

View File

@@ -0,0 +1,48 @@
import logging
from fastapi import APIRouter, HTTPException
from twitch_vod.services.db.repository.channel import (
create_channel,
delete_channel,
get_all_channels,
)
from twitch_vod.services.twitch.controller import TwitchController
from twitch_vod.services.twitch.models.errors import ChannelDoesNotExist
router = APIRouter(prefix="/channels")
logger = logging.getLogger(__name__)
@router.post("/{name}", status_code=201)
async def handle_create_channel(name: str):
try:
await TwitchController().add_client(name)
create_channel(name)
except ChannelDoesNotExist:
logger.info(f"channel does not exist: {name}")
raise HTTPException(status_code=404, detail="channel does not exist")
except Exception as e:
logger.error(f"error creating the channel {e}")
raise HTTPException(status_code=500, detail="failed to create the channel")
@router.delete("/{name}", status_code=200)
def handle_delete_channel(name: str):
try:
delete_channel(name)
TwitchController().remove_client(name)
except ChannelDoesNotExist:
raise HTTPException(status_code=404, detail="channel was not found to delete")
except Exception as e:
logger.error(f"failed to delete the channel: {e}")
raise HTTPException(status_code=500, detail="failed to delete the channel")
@router.get("/", status_code=200)
def handle_get_all_channels():
try:
channels = get_all_channels()
return {"channels": channels}
except Exception as e:
logger.error(f"failed to get all channels {e}")
raise HTTPException(status_code=500, detail="failed to get all channels")

View File

@@ -0,0 +1,33 @@
from boto3 import client
from botocore.exceptions import ClientError
from twitch_vod.config import Config
class S3Bucket:
def __init__(self):
self._client = None
self.config = Config()
def _get_client(self):
if self._client is None:
self._client = client(
"s3",
endpoint_url=self.config.S3_ENDPOINT_URL,
aws_access_key_id=self.config.S3_ACCESS_KEY,
aws_secret_access_key=self.config.S3_SECRET_KEY,
region_name=self.config.S3_REGION,
)
return self._client
def store_file(self, key: str, file_path: str, bucket_name: str):
try:
self._get_client().upload_file(file_path, bucket_name, key)
except ClientError as e:
raise RuntimeError(f"Failed to upload file: {e}")
def get_file(self, key: str, file_path: str, bucket_name: str):
try:
self._get_client().download_file(bucket_name, key, file_path)
except ClientError as e:
raise RuntimeError(f"Failed to download file: {e}")

View File

@@ -0,0 +1,5 @@
from sqlmodel import Field, SQLModel
class Channel(SQLModel, table=True):
channel_name: str = Field(primary_key=True)

View File

@@ -0,0 +1,8 @@
from uuid import uuid4
from sqlmodel import UUID, Field, SQLModel
class VOD(SQLModel, table=True):
uuid: UUID = Field(default=uuid4(), primary_key=True)
vod_bucket_key: str
vod_bucket_message_key: str

View File

@@ -0,0 +1,23 @@
import logging
from sqlmodel import create_engine
from twitch_vod.config import Config
logger = logging.getLogger(__name__)
config = Config()
def postgres_engine():
try:
connection_string = (
f"postgresql://{config.POSTGRES_USER}:{config.POSTGRES_PASSWORD}"
f"@{config.POSTGRES_HOST}:{config.POSTGRES_PORT}/{config.POSTGRES_DB}"
)
engine = create_engine(connection_string)
logger.info(f"successfully created the postgres engine: {engine}")
return engine
except Exception as e:
logger.error(f"failed to create the postgres engine: {e}")
raise RuntimeError(e)

View File

@@ -0,0 +1,45 @@
import logging
from sqlmodel import Session, select
from twitch_vod.services.db.models.channel import Channel
from twitch_vod.services.db.postgres import postgres_engine
from twitch_vod.services.twitch.models.errors import ChannelDoesNotExist
logger = logging.getLogger(__name__)
def create_channel(channel_name):
channel = Channel(channel_name=channel_name)
session = Session(postgres_engine())
session.add(channel)
session.commit()
session.refresh(channel)
return channel
def delete_channel(channel_name: str):
session = Session(postgres_engine())
statement = select(Channel).where(Channel.channel_name == channel_name)
result = session.exec(statement)
try:
channel = result.one()
session.delete(channel)
session.commit()
except Exception as e:
logger.info("failed to find channel to delete")
ChannelDoesNotExist(e)
def get_all_channels():
session = Session(postgres_engine())
statement = select(Channel)
result = session.exec(statement)
channels = result.all()
return channels

View File

@@ -0,0 +1,27 @@
from uuid import UUID
from sqlmodel import Session, select
from twitch_vod.services.db.models.vod import VOD
from twitch_vod.services.db.postgres import postgres_engine
def create_vod(bucket_video_key: str, bucket_msg_key: str):
vod = VOD(vod_bucket_key=bucket_video_key, vod_bucket_message_key=bucket_msg_key)
session = Session(postgres_engine())
session.add(vod)
session.commit()
session.refresh(vod)
return vod
def delete_vod(uuid: UUID):
session = Session(postgres_engine())
statement = select(VOD).where(VOD.uuid == uuid)
result = session.exec(statement)
vod = result.one()
session.delete(vod)
session.commit()

View File

@@ -0,0 +1,78 @@
from typing import Dict
from loguru import logger
from twitchAPI.twitch import Twitch
from twitch_vod.config import Config
from twitch_vod.services.twitch.models.errors import ChannelDoesNotExist
from .twitch import _TwitchClient
class TwitchController:
_instance = None
_lock = __import__("threading").Lock()
def __new__(cls):
if cls._instance is None:
with cls._lock:
if cls._instance is None:
cls._instance = super().__new__(cls)
return cls._instance
def __init__(self) -> None:
if not hasattr(self, "_initialized"):
self._clients: Dict[str, _TwitchClient] = {}
self._config = Config()
self._initialized = True
self._twitch_api_client = None
async def create(self):
self._twitch_api_client = await Twitch(
Config().TWITCH_CLIENT_ID, # type: ignore
Config().TWITCH_SECRET, # type: ignore
)
return self
async def add_client(self, channel_name: str):
if self._twitch_api_client is None:
raise RuntimeError(
"need to use the `create` method when creating the TwitchController"
)
twitch_client = _TwitchClient(channel_name, self._twitch_api_client)
if not await self.real_channel(channel_name):
raise ChannelDoesNotExist("twitch channel not found")
try:
await twitch_client.connect()
self._clients[channel_name] = twitch_client
except Exception:
logger.error(f"failed to add client {twitch_client}")
async def real_channel(self, channel_name: str) -> bool:
try:
users = await self._twitch_api_client.get_users(logins=[channel_name]) # type: ignore
return len(users) > 0
except Exception:
return False
def remove_client(self, channel_name: str):
twitch_client = self._clients.get(channel_name)
if twitch_client is None:
logger.warning(
f"tried to removed twitch client with channel name {channel_name}, however, it does not exist"
)
return
try:
twitch_client.disconnect()
except Exception:
logger.error(f"failed to discconect client {twitch_client}")
del self._clients[channel_name]
def cleanup(self):
for _, twitch_client in self._clients.items():
twitch_client.disconnect()

View File

@@ -0,0 +1,16 @@
class IRCError(Exception):
"""Error when communication with an IRC fails"""
pass
class UnexpectedError(Exception):
"""Catch all errors for errors we did not expect"""
pass
class ChannelDoesNotExist(Exception):
"""Channel does not exist"""
pass

View File

@@ -0,0 +1,8 @@
from pydantic import BaseModel
from datetime import datetime
class Message(BaseModel):
username: str
message: str
timestamp: datetime

View File

@@ -0,0 +1,223 @@
import socket
import time
import threading
import subprocess
from pathlib import Path
from loguru import logger
from collections import deque
from typing import Optional
from datetime import datetime
from twitchAPI.twitch import Twitch
from twitchAPI.oauth import UserAuthenticator
from twitchAPI.chat import Chat, ChatEvent, ChatMessage, EventData
from twitchAPI.type import AuthScope
from .models.errors import IRCError, UnexpectedError
class _TwitchClient:
async def __init__(self, channel: str, twitch_api: Twitch) -> None:
self._twitch_api = twitch_api
self._chat: Optional[Chat] = None
self._socket: Optional[socket.socket] = None
self._message_timestamps: deque[float] = deque()
self._start_time = datetime.now()
self._total_messages = 0
self._connected = False
self._channel = channel
self._failed_socket_attempts = 0
self._recording_thread: Optional[threading.Thread] = None
self._is_recording = False
self._should_stop_recording = False
self._current_vod_path: Optional[str] = None
self._recording_process: Optional[subprocess.Popen] = None
self._streamlink_process: Optional[subprocess.Popen] = None
async def _get_user_access_token(self):
auth = UserAuthenticator(self._twitch_api, [AuthScope.CHAT_READ])
token, refresh_token = await auth.authenticate() # type: ignore
await self._twitch_api.set_user_authentication(
token, [AuthScope.CHAT_READ], refresh_token
)
def get_connection_status(self):
return self._connected
def get_channel(self):
return self._channel
def get_total_messages(self):
return self._total_messages
def get_start_time(self):
return self._start_time
def start_recording_stream(self):
"""Start recording the stream from twitch"""
if self._is_recording:
raise RuntimeError("Recording is already in progress")
self._is_recording = True
self._should_stop_recording = False
self._recording_thread = threading.Thread(
target=self._record_stream_thread, daemon=True
)
self._recording_thread.start()
logger.info(f"Started recording stream for channel {self._channel}")
return True
async def connect(self):
"""High level connection function"""
logger.info(f"starting connection to Twitch channel {self._channel}")
try:
await self.irc_connect()
self.start_recording_stream()
logger.info(
f"successfully disconnected from twitch channel {self._channel}"
)
except Exception as e:
logger.error(f"failed to connect to twitch channel {self._channel}")
raise Exception(e)
def disconnect(self):
"""High level disconnect function"""
logger.info(f"starting disconnecting from twitch channel {self._channel}")
try:
self.irc_disconnect()
self.stop_recording_stream()
logger.info(
f"successfully disconnected from twitch channel {self._channel}"
)
except Exception as e:
logger.error(f"failed to disconnect from twitch channel {self._channel}")
raise Exception(e)
def stop_recording_stream(self):
"""Stops recording the stream from twitch"""
if not self._is_recording:
raise RuntimeError("No recording is in progress")
self._should_stop_recording = True
if self._streamlink_process:
logger.info("stopping stream link")
self._streamlink_process.kill()
self._streamlink_process = None
logger.info("success stopping stream link")
if self._recording_process:
logger.info("stopping ffmpeg")
self._recording_process.kill()
self._recording_process = None
logger.info("success stopping ffmpeg")
if self._recording_thread:
logger.info("waiting for recording thread to join back in")
self._recording_thread.join()
logger.info("threaded joined")
self._recording_thread = None
self._is_recording = False
self._recording_thread = None
logger.info(f"Stopped recording stream for channel {self._channel}")
return True
def _path(self):
return Path(f"vods/{self._channel}_{self._start_time}/")
def _vod_path(self):
vods_dir = self._path()
vods_dir.mkdir(exist_ok=True)
vod_filename = f"{self._channel}_{self._start_time}.mkv"
return vods_dir / vod_filename
def _message_path(self):
msg_dir = self._path()
msg_dir.mkdir(exist_ok=True)
return msg_dir / "messages.txt"
def _record_stream_thread(self):
twitch_url = f"https://twitch.tv/{self._channel}"
self._current_vod_path = str(self._vod_path())
logger.info(f"Recording to file: {self._current_vod_path}")
streamlink_cmd = [
"streamlink",
twitch_url,
"best",
"--stdout",
"--twitch-supported-codecs",
"av1,h265",
]
ffmpeg_cmd = [
"ffmpeg",
"-i",
"pipe:0",
"-vf",
"fps=24",
"-c:v",
"av1_nvenc",
"-preset",
"fast",
"-crf",
"23",
"-y",
self._current_vod_path,
]
self._streamlink_process = subprocess.Popen(
streamlink_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
self._recording_process = subprocess.Popen(
ffmpeg_cmd, stdin=self._streamlink_process.stdout, stderr=subprocess.PIPE
)
if self._streamlink_process.stdout:
self._streamlink_process.stdout.close()
while True:
if self._should_stop_recording:
logger.info("Stop signal received, terminating recording")
break
time.sleep(1)
async def irc_connect(self):
self._chat = await Chat(self._twitch_api)
self._chat.register_event(ChatEvent.READY, self._chat_on_ready)
self._chat.register_event(ChatEvent.MESSAGE, self._chat_on_message)
self._chat.start()
async def _chat_on_ready(self, ready_event: EventData):
await ready_event.chat.join_room(self._channel)
async def _chat_on_message(self, msg: ChatMessage):
self._write_messages(f"{msg.user.name}: {msg.text}: {msg.sent_timestamp}")
def irc_disconnect(self):
if self._chat is None:
return
self._chat.stop()
def _write_messages(self, message: str):
"""Takes a list of twitch irc messages"""
msg_path = self._message_path()
msg_path.touch(exist_ok=True)
with open(msg_path, "a") as f:
f.write(f"{message}\n")

View File

@@ -0,0 +1,109 @@
from config import config_types, Config
from textual.app import App, ComposeResult
from textual.containers import Horizontal, Vertical
from textual.widgets import Input, Log, Button
import asyncio
# this directly accesses the underlying Twitch client, since this
# is only intended to manually test the client
from services.twitch.twitch import _TwitchClient as TwitchClient
class TwitchTUI(App):
CSS = """
.header {
height: 3;
dock: top;
}
.main-content {
height: 1fr;
}
#channel-input {
width: 1fr;
}
#connect-btn {
margin-left: 1;
}
#log {
height: 1fr;
border-top: solid $primary;
}
"""
def __init__(self, config: Config) -> None:
super().__init__()
self.twitch_service = TwitchClient(
config.TWITCH_ACCESS_TOKEN, config.TWITCH_USERNAME, ""
)
def compose(self) -> ComposeResult:
with Vertical(classes="main-content"):
with Horizontal(classes="header"):
yield Input(placeholder="Enter channel name...", id="channel-input")
yield Button("Connect", id="connect-btn")
yield Button("Disconnect", id="disconnect-btn")
yield Log(id="log")
async def update(self):
while self.twitch_service.get_connection_status():
messages = self.twitch_service.read_messages()
for m in messages:
self.add_log(m)
await asyncio.sleep(1)
self.add_log("worker completed")
def on_button_pressed(self, event: Button.Pressed) -> None:
if event.button.id == "connect-btn":
channel_input = self.query_one("#channel-input", Input)
channel = channel_input.value.strip()
if channel:
self.log(f"Connecting to channel: {channel}")
self.start(channel)
else:
self.log("Please enter a channel name", "warning")
elif event.button.id == "disconnect-btn":
self.log("Disconnection from channel")
self.stop()
def on_input_submitted(self, event: Input.Submitted) -> None:
if event.input.id == "channel-input":
channel = event.input.value.strip()
if channel:
self.log(f"Connecting to channel: {channel}")
event.input.value = ""
else:
self.log("Please enter a channel name", "warning")
def start(self, channel: str):
self.add_log("Connecting...")
try:
self.twitch_service._channel = channel
self.add_log("starting irc connection")
self.twitch_service.irc_connect()
self.add_log("successfully connected")
self.add_log("starting stream recording")
self.twitch_service.start_recording_stream()
self.add_log("successfully connected")
self.run_worker(self.update)
except Exception as e:
self.add_log(f"failed to connect: {e}")
self.stop()
def stop(self):
try:
self.twitch_service.irc_disconnect()
self.twitch_service.stop_recording_stream()
except Exception:
pass
def add_log(self, message: str, level: str = "info") -> None:
log_widget = self.query_one("#log", Log)
log_widget.write_line(f"[{level.upper()}] {message}")
if __name__ == "__main__":
TwitchTUI(config_types["development"]).run()

View File