diff --git a/twitch-highlight/.gitignore b/twitch-highlight/.gitignore new file mode 100644 index 0000000..706abfd --- /dev/null +++ b/twitch-highlight/.gitignore @@ -0,0 +1,70 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db diff --git a/twitch-highlight/README.md b/twitch-highlight/README.md new file mode 100644 index 0000000..e7a65b4 --- /dev/null +++ b/twitch-highlight/README.md @@ -0,0 +1,72 @@ +# Twitch Highlight + +A Twitch highlight extraction tool for extracting clips and segments from Twitch streams. + +## Configuration + +This application requires Twitch API credentials to function. You need to set up a `.env` file in the root of the project with the following environment variables: + +``` +TWITCH_NICKNAME=your_twitch_username +TWITCH_OAUTH_TOKEN=oauth:your_oauth_token +TWITCH_CLIENT_ID=your_client_id +``` + +### Getting Twitch Credentials + +You have a few options to get your OAuth token: + +1. Go to [https://twitchtokengenerator.com/](https://twitchtokengenerator.com/) +2. Click "Connect with Twitch" to authorize +3. Select the scopes you need (e.g., "Chat: Read Chat", "Chat: Send Chat Messages") +4. Copy the generated token (it will start with `oauth:`) + +#### 4. Your Twitch Nickname + +This is simply your Twitch username (the one you use to log in). For example, if your Twitch channel is `https://twitch.tv/yourname`, your nickname is `yourname`. + +### Setting Up the .env File + +Create a `.env` file in the project root directory: + +```env +TWITCH_NICKNAME=relicjamin1 +TWITCH_OAUTH_TOKEN=oauth:your_actual_token_here +TWITCH_CLIENT_ID=your_actual_client_id_here +``` + +**Important**: Never commit your `.env` file to version control. The `.env` file is already included in `.gitignore`. + +## Installation + +```bash +pip install -e . +``` + +For development dependencies: + +```bash +pip install -e ".[dev]" +``` + +## Usage + +```bash +python ./src/main.py +``` + +## Development + +```bash +# Run tests +pytest + +# Format code +black . + +# Lint +ruff check . + +# Type check +mypy src/ +``` diff --git a/twitch-highlight/pyproject.toml b/twitch-highlight/pyproject.toml new file mode 100644 index 0000000..16aea90 --- /dev/null +++ b/twitch-highlight/pyproject.toml @@ -0,0 +1,69 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "twitch-highlight" +version = "0.1.0" +description = "Twitch highlight extraction tool" +readme = "README.md" +requires-python = ">=3.8" +license = {text = "MIT"} +authors = [ + {name = "Collin Campbell", email = "collin@example.com"} +] +keywords = ["twitch", "highlight", "video"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = ["python-dotenv>=1.0.0", "streamlink>=6.0.0", "textual>=7.5.0"] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0", + "pytest-cov>=4.0", + "black>=23.0", + "ruff>=0.1.0", + "mypy>=1.0", + "textual-dev>=1.8.0" +] + +[project.urls] +Homepage = "https://github.com/collincampbell/twitch-highlight" +Repository = "https://github.com/collincampbell/twitch-highlight" +Issues = "https://github.com/collincampbell/twitch-highlight/issues" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.black] +line-length = 100 +target-version = ['py38', 'py39', 'py310', 'py311', 'py312'] + +[tool.ruff] +line-length = 100 +select = ["E", "F", "I", "N", "W"] +ignore = [] + +[tool.ruff.per-file-ignores] +"__init__.py" = ["F401"] + +[tool.mypy] +python_version = "3.8" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] diff --git a/twitch-highlight/src/__init__.py b/twitch-highlight/src/__init__.py new file mode 100644 index 0000000..c6e840b --- /dev/null +++ b/twitch-highlight/src/__init__.py @@ -0,0 +1,3 @@ +"""Twitch Highlight Package.""" + +__version__ = "0.1.0" diff --git a/twitch-highlight/src/main.py b/twitch-highlight/src/main.py new file mode 100644 index 0000000..8f2507c --- /dev/null +++ b/twitch-highlight/src/main.py @@ -0,0 +1,13 @@ +"""Main module for twitch-highlight.""" + +from services.config import load_config +from tui.tui import TwitchHighlightTUI + + +def main(): + """Main entry point.""" + TwitchHighlightTUI(load_config()).run() + + +if __name__ == "__main__": + main() diff --git a/twitch-highlight/src/services/config.py b/twitch-highlight/src/services/config.py new file mode 100644 index 0000000..dd5500c --- /dev/null +++ b/twitch-highlight/src/services/config.py @@ -0,0 +1,33 @@ +import os +from pathlib import Path + +from dotenv import load_dotenv + +env_path = Path(__file__).parent.parent.parent / ".env" +load_dotenv(env_path) + +TWITCH_NICKNAME = os.getenv("TWITCH_NICKNAME", "") +TWITCH_OAUTH_TOKEN = os.getenv("TWITCH_OAUTH_TOKEN", "") +TWITCH_CLIENT_ID = os.getenv("TWITCH_CLIENT_ID", "") + + +class Config: + def __init__(self, nickname: str, oauth_token: str, client_id: str): + self.nickname = nickname + self.oauth_token = oauth_token + self.client_id = client_id + + +def load_config() -> Config: + missing = [] + if not TWITCH_NICKNAME: + missing.append("TWITCH_NICKNAME") + if not TWITCH_OAUTH_TOKEN: + missing.append("TWITCH_OAUTH_TOKEN") + if not TWITCH_CLIENT_ID: + missing.append("TWITCH_CLIENT_ID") + + if missing: + raise ValueError(f"Missing required environment variables: {', '.join(missing)}") + + return Config(TWITCH_NICKNAME, TWITCH_OAUTH_TOKEN, TWITCH_CLIENT_ID) diff --git a/twitch-highlight/src/services/twitch_client.py b/twitch-highlight/src/services/twitch_client.py new file mode 100644 index 0000000..7badb3b --- /dev/null +++ b/twitch-highlight/src/services/twitch_client.py @@ -0,0 +1,115 @@ +import socket +import time + +from dataclasses import dataclass +from collections import deque +from typing import Optional + + +@dataclass +class ChatRateStats: + current_rate: float + average_rate: float + total_messages: int + rate_of_change: float + + +class TwitchClient: + def __init__(self, oauth_token: str, nickname: str, window_seconds: int = 60): + self.oauth_token = oauth_token + self.nickname = nickname + self.window_seconds = window_seconds + self.socket: Optional[socket.socket] = None + self.message_timestamps: deque[float] = deque() + self.start_time: Optional[float] = None + self.total_messages = 0 + self.connected = False + + self.channel = "" + + def set_channel(self, channel: str): + self.channel = channel + + def connect(self) -> None: + self.socket = socket.socket() + self.socket.connect(("irc.chat.twitch.tv", 6667)) + self.socket.send(f"PASS {self.oauth_token}\n".encode("utf-8")) + self.socket.send(f"NICK {self.nickname}\n".encode("utf-8")) + self.socket.send(f"JOIN #{self.channel}\n".encode("utf-8")) + self.connected = True + self.start_time = time.time() + + def disconnect(self) -> None: + if self.socket: + self.socket.send("PART #{}\n".format(self.channel).encode("utf-8")) + self.socket.close() + self.connected = False + + def _receive_messages(self) -> list[str]: + messages = [] + if self.socket: + try: + data = self.socket.recv(2048).decode("utf-8") + print(data) + + if data.find("PING") != -1: + print("sending back pong") + self.socket.send("PONG :tmi.twitch.tv\n".encode("utf-8")) + + messages = [line for line in data.split("\r\n") if line.strip()] + except Exception: + pass + return messages + + def _is_chat_message(self, line: str) -> bool: + return "PRIVMSG #{} :".format(self.channel) in line + + def _cleanup_old_timestamps(self) -> None: + current_time = time.time() + while ( + self.message_timestamps + and current_time - self.message_timestamps[0] > self.window_seconds + ): + self.message_timestamps.popleft() + + def read_messages(self) -> list[str]: + lines = self._receive_messages() + chat_messages = [] + current_time = time.time() + + for line in lines: + if self._is_chat_message(line): + chat_messages.append(line) + self.message_timestamps.append(current_time) + self.total_messages += 1 + + self._cleanup_old_timestamps() + return chat_messages + + def _calculate_current_rate(self) -> float: + if not self.message_timestamps: + return 0.0 + return len(self.message_timestamps) / self.window_seconds + + def _calculate_average_rate(self) -> float: + if not self.start_time: + return 0.0 + elapsed_time = time.time() - self.start_time + if elapsed_time == 0: + return 0.0 + return self.total_messages / elapsed_time + + def get_rate_stats(self, old: ChatRateStats) -> ChatRateStats: + self._cleanup_old_timestamps() + current_rate = self._calculate_current_rate() + average_rate = self._calculate_average_rate() + + roc = 0.0 + if old.current_rate != 0.0: + roc = (current_rate - old.current_rate) / old.current_rate + return ChatRateStats( + current_rate=current_rate, + average_rate=average_rate, + total_messages=self.total_messages, + rate_of_change=roc, + ) diff --git a/twitch-highlight/src/services/twitch_video.py b/twitch-highlight/src/services/twitch_video.py new file mode 100644 index 0000000..b92ef74 --- /dev/null +++ b/twitch-highlight/src/services/twitch_video.py @@ -0,0 +1,119 @@ +import io +import json +import subprocess +import urllib.request +from pathlib import Path +from typing import Callable, Optional + + +class TwitchVideoClient: + def __init__(self, oauth_token: str, client_id: str, buffer_seconds: int = 15): + self.oauth_token = oauth_token + self.client_id = client_id + self.base_url = "https://api.twitch.tv/helix" + self.buffer_seconds = buffer_seconds + self._buffer: io.BytesIO = io.BytesIO() + self._is_recording = False + self._process: Optional[subprocess.Popen] = None + + self.channel = "" + + def set_channel(self, channel: str): + self.channel = channel + + def _make_request(self, endpoint: str, params: Optional[dict] = None) -> dict: + url = f"{self.base_url}{endpoint}" + if params: + url += f"?{'&'.join(f'{k}={v}' for k, v in params.items())}" + + req = urllib.request.Request(url) + req.add_header("Authorization", f"Bearer {self.oauth_token.removeprefix('oauth:')}") + req.add_header("Client-Id", self.client_id) + + with urllib.request.urlopen(req) as response: + return json.loads(response.read().decode("utf-8")) + + def _get_stream_url(self) -> str: + try: + import streamlink + + streams = streamlink.streams(f"https://twitch.tv/{self.channel}") + if "best" not in streams: + raise ValueError(f"No stream available for channel {self.channel}") + return streams["best"].url + except ImportError: + raise ImportError("streamlink is required. Install with: pip install streamlink") + except Exception as e: + raise RuntimeError(f"Failed to get stream URL: {e}") + + def start_recording(self, on_chunk: Optional[Callable[[bytes], None]] = None) -> None: + if self._is_recording: + raise RuntimeError("Already recording") + + try: + stream_url = self._get_stream_url() + + ffmpeg_cmd = [ + "ffmpeg", + "-i", + stream_url, + "-t", + str(self.buffer_seconds), + "-c", + "copy", + "-f", + "mpegts", + "-", + ] + + self._process = subprocess.Popen( + ffmpeg_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + self._is_recording = True + self._buffer = io.BytesIO() + + def read_stream(): + try: + while True: + chunk = self._process.stdout.read(8192) + if not chunk: + break + self._buffer.write(chunk) + if on_chunk: + on_chunk(chunk) + except Exception: + pass + + import threading + + self._read_thread = threading.Thread(target=read_stream) + self._read_thread.daemon = True + self._read_thread.start() + + except Exception as e: + raise RuntimeError(f"Failed to start recording: {e}") + + def stop_recording(self) -> bytes: + if not self._is_recording: + raise RuntimeError("Not recording") + + if self._process: + self._process.terminate() + self._process.wait() + + self._is_recording = False + self._buffer.seek(0) + return self._buffer.getvalue() + + def save_to_file(self, filepath: Path, data: Optional[bytes] = None) -> Path: + if data is None: + data = self._buffer.getvalue() + + filepath.parent.mkdir(parents=True, exist_ok=True) + with open(filepath, "wb") as f: + f.write(data) + return filepath + + def clear_buffer(self) -> None: + self._buffer = io.BytesIO() diff --git a/twitch-highlight/src/tui/tui.py b/twitch-highlight/src/tui/tui.py new file mode 100644 index 0000000..2d37821 --- /dev/null +++ b/twitch-highlight/src/tui/tui.py @@ -0,0 +1,114 @@ +from services.config import Config +from services.twitch_client import TwitchClient +from services.twitch_video import TwitchVideoClient +from textual.app import App, ComposeResult +from textual.containers import Horizontal, Vertical +from textual.widgets import Input, Log, Button +import time + +import datetime +from pathlib import Path + +from services.twitch_client import ChatRateStats +import asyncio + + +class TwitchHighlightTUI(App): + CSS = """ + .header { + height: 3; + dock: top; + } + .main-content { + height: 1fr; + } + #channel-input { + width: 1fr; + } + #connect-btn { + margin-left: 1; + } + #log { + height: 1fr; + border-top: solid $primary; + } + """ + + def __init__(self, config: Config) -> None: + super().__init__() + self.twitch_chat = TwitchClient(config.oauth_token, config.nickname) + self.twitch_record = TwitchVideoClient(config.oauth_token, config.client_id) + + def compose(self) -> ComposeResult: + with Vertical(classes="main-content"): + with Horizontal(classes="header"): + yield Input(placeholder="Enter channel name...", id="channel-input") + yield Button("Connect", id="connect-btn") + yield Button("Disconnect", id="disconnect-btn") + yield Log(id="log") + + async def update(self): + start_time = time.time() + + current_rate = ChatRateStats( + current_rate=0.0, average_rate=0.0, total_messages=0, rate_of_change=0.0 + ) + while self.twitch_chat.connected: + messages = self.twitch_chat.read_messages() + current_rate = self.twitch_chat.get_rate_stats(current_rate) + + self.add_log(f"{current_rate}, highlights enabled: {time.time() - start_time > 120}") + for m in messages: + self.add_log(m) + if current_rate.rate_of_change > 0.8 and time.time() - start_time > 120: + self.add_log("POG CHAMP HIGHLIGH TIME") + self.twitch_record.stop_recording() + self.twitch_record.save_to_file( + Path(f"clips/{self.twitch_record.channel}-{datetime.datetime.now()}.mp4") + ) + self.twitch_record.clear_buffer() + self.twitch_record.start_recording() + await asyncio.sleep(15) + + self.add_log("disconnected worker") + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == "connect-btn": + channel_input = self.query_one("#channel-input", Input) + channel = channel_input.value.strip() + if channel: + self.log(f"Connecting to channel: {channel}") + self.twitch_chat.set_channel(channel) + self.twitch_record.set_channel(channel) + self.start_twitch() + else: + self.log("Please enter a channel name", "warning") + elif event.button.id == "disconnect-btn": + self.log("Disconnection from channel") + self.twitch_chat.set_channel("") + self.twitch_chat.set_channel("") + self.dissconnect_twitch() + + def on_input_submitted(self, event: Input.Submitted) -> None: + if event.input.id == "channel-input": + channel = event.input.value.strip() + if channel: + self.log(f"Connecting to channel: {channel}") + event.input.value = "" + else: + self.log("Please enter a channel name", "warning") + + def start_twitch(self): + self.add_log("Connecting...") + self.twitch_chat.connect() + self.twitch_record.start_recording() + self.run_worker(self.update) + + def dissconnect_twitch(self): + self.add_log("diconnecting, wait until worker reports its done") + self.twitch_chat.disconnect() + self.twitch_record.stop_recording() + + def add_log(self, message: str, level: str = "info") -> None: + log_widget = self.query_one("#log", Log) + log_widget.write_line(f"[{level.upper()}] {message}") diff --git a/twitch-highlight/tests/__init__.py b/twitch-highlight/tests/__init__.py new file mode 100644 index 0000000..651995d --- /dev/null +++ b/twitch-highlight/tests/__init__.py @@ -0,0 +1,6 @@ +"""Tests package.""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) diff --git a/twitch-highlight/tests/test_example.py b/twitch-highlight/tests/test_example.py new file mode 100644 index 0000000..c903546 --- /dev/null +++ b/twitch-highlight/tests/test_example.py @@ -0,0 +1,8 @@ +"""Example test module.""" + +import pytest + + +def test_example(): + """Example test function.""" + assert True diff --git a/twitch-vod/.env.example b/twitch-vod/.env.example new file mode 100644 index 0000000..0ee176e --- /dev/null +++ b/twitch-vod/.env.example @@ -0,0 +1,10 @@ +# Twitch API credentials +TWITCH_CLIENT_ID=your-twitch-client-id +TWITCH_ACCESS_TOKEN=your-twitch-access-token + +# Postgres configuration +POSTGRES_HOST=localhost +POSTGRES_PORT=5432 +POSTGRES_DB=twitch_vod +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres diff --git a/twitch-vod/.gitignore b/twitch-vod/.gitignore new file mode 100644 index 0000000..3ca9bef --- /dev/null +++ b/twitch-vod/.gitignore @@ -0,0 +1,146 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +Pipfile.lock + +# poetry +poetry.lock + +# pdm +.pdm.toml + +# PEP 582 +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# IDEs +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Project specific +vods/ diff --git a/twitch-vod/Makefile b/twitch-vod/Makefile new file mode 100644 index 0000000..8b05e7b --- /dev/null +++ b/twitch-vod/Makefile @@ -0,0 +1,8 @@ +.PHONY: tui + +tui: + python ./src/twitch_vod/tui.py + +.PHONY: run +run: + fastapi dev ./src/twitch_vod/main.py diff --git a/twitch-vod/README.md b/twitch-vod/README.md new file mode 100644 index 0000000..e69de29 diff --git a/twitch-vod/docker-compose.yml b/twitch-vod/docker-compose.yml new file mode 100644 index 0000000..11ef073 --- /dev/null +++ b/twitch-vod/docker-compose.yml @@ -0,0 +1,47 @@ +services: + postgres: + image: postgres:16-alpine + container_name: twitch-vod-postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: twitch_vod + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + + pgadmin: + image: dpage/pgadmin4:latest + container_name: twitch-vod-pgadmin + environment: + PGADMIN_DEFAULT_EMAIL: admin@admin.com + PGADMIN_DEFAULT_PASSWORD: admin + PGADMIN_LISTEN_PORT: 80 + ports: + - "5050:80" + depends_on: + postgres: + condition: service_healthy + + minio: + image: minio/minio:latest + container_name: twitch-vod-minio + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + ports: + - "9000:9000" + - "9001:9001" + volumes: + - minio_data:/data + command: server /data --console-address ":9001" + +volumes: + postgres_data: + minio_data: diff --git a/twitch-vod/pyproject.toml b/twitch-vod/pyproject.toml new file mode 100644 index 0000000..d251b22 --- /dev/null +++ b/twitch-vod/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "twitch-vod" +version = "0.1.0" +description = "" +authors = [ + {name = "Collin Campbell",email = "collincampbell97@proton.me"} +] +readme = "README.md" +requires-python = ">=3.10,<4" + dependencies = [ + "streamlink (>=8.1.2,<9.0.0)", + "twitchapi (>=4.5.0,<5.0.0)", + "dotenv (>=0.9.9,<0.10.0)", + "loguru (>=0.7.3,<0.8.0)", + "pydantic (>=2.12.5,<3.0.0)", + "textual (>=7.5.0,<8.0.0)", + "textual-dev (>=1.8.0,<2.0.0)", + "psycopg2-binary (>=2.9.9,<3.0.0)", + "boto3 (>=1.35.0,<2.0.0)", + "fastapi[standard] (>=0.128.5,<0.129.0)", + ] + +[tool.poetry] +packages = [{include = "twitch_vod", from = "src"}] + + +[build-system] +requires = ["poetry-core>=2.0.0,<3.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/twitch-vod/src/twitch_vod/__init__.py b/twitch-vod/src/twitch_vod/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/twitch-vod/src/twitch_vod/config.py b/twitch-vod/src/twitch_vod/config.py new file mode 100644 index 0000000..d896b3a --- /dev/null +++ b/twitch-vod/src/twitch_vod/config.py @@ -0,0 +1,61 @@ +import os + +from dotenv import load_dotenv + +load_dotenv() + +basedir = os.path.abspath(os.path.dirname(__file__)) + + +class Config(object): + TWITCH_CLIENT_ID = os.getenv("TWITCH_CLIENT_ID") + TWITCH_SECRET = os.getenv("TWITCH_SECRET") + + POSTGRES_HOST = os.getenv("POSTGRES_HOST") + POSTGRES_PORT = os.getenv("POSTGRES_PORT") + POSTGRES_DB = os.getenv("POSTGRES_DB") + POSTGRES_USER = os.getenv("POSTGRES_USER") + POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD") + + S3_ENDPOINT_URL = os.getenv("S3_ENDPOINT_URL") + S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY") + S3_SECRET_KEY = os.getenv("S3_SECRET_KEY") + S3_REGION = os.getenv("S3_REGION", "us-east-1") + + def __init__(self): + required_vars = [ + "TWITCH_CLIENT_ID", + "TWITCH_SECRET", + "POSTGRES_HOST", + "POSTGRES_PORT", + "POSTGRES_DB", + "POSTGRES_USER", + "POSTGRES_PASSWORD", + "S3_ENDPOINT_URL", + "S3_ACCESS_KEY", + "S3_SECRET_KEY", + ] + missing = [var for var in required_vars if not os.getenv(var)] + + if missing: + raise ValueError( + f"Missing required environment variables: {', '.join(missing)}" + ) + + +class DevelopmentConfig(Config): + DEBUG = True + LOG_BACKTRACE = True + LOG_LEVEL = "DEBUG" + + +class ProductionConfig(Config): + LOG_BACKTRACE = False + LOG_LEVEL = "INFO" + + +config_types = { + "development": DevelopmentConfig, + "production": ProductionConfig, + "default": DevelopmentConfig, +} diff --git a/twitch-vod/src/twitch_vod/main.py b/twitch-vod/src/twitch_vod/main.py new file mode 100644 index 0000000..2d04a72 --- /dev/null +++ b/twitch-vod/src/twitch_vod/main.py @@ -0,0 +1,37 @@ +import logging + +# need to import the models to allow SQLModel to create the DB tables for us +from twitch_vod.services.db.models import * # noqa # pylint: disable=unused-import +from contextlib import asynccontextmanager +from fastapi import FastAPI +from sqlmodel import SQLModel + +from twitch_vod.services.db.postgres import postgres_engine +from twitch_vod.routers import channel +from twitch_vod.services.twitch.controller import TwitchController + +logger = logging.getLogger(__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + logger.info("application starting") + + engine = postgres_engine() + twitch_controller = await TwitchController().create() + SQLModel.metadata.create_all(engine) + + yield + + logger.info("application is shutting down") + engine.dispose() + twitch_controller.cleanup() + + +app = FastAPI(lifespan=lifespan) +app.include_router(channel.router) + + +@app.get("/") +async def root(): + return {"healthy"} diff --git a/twitch-vod/src/twitch_vod/routers/__init__.py b/twitch-vod/src/twitch_vod/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/twitch-vod/src/twitch_vod/routers/channel.py b/twitch-vod/src/twitch_vod/routers/channel.py new file mode 100644 index 0000000..c3f6761 --- /dev/null +++ b/twitch-vod/src/twitch_vod/routers/channel.py @@ -0,0 +1,48 @@ +import logging +from fastapi import APIRouter, HTTPException + +from twitch_vod.services.db.repository.channel import ( + create_channel, + delete_channel, + get_all_channels, +) +from twitch_vod.services.twitch.controller import TwitchController +from twitch_vod.services.twitch.models.errors import ChannelDoesNotExist + +router = APIRouter(prefix="/channels") +logger = logging.getLogger(__name__) + + +@router.post("/{name}", status_code=201) +async def handle_create_channel(name: str): + try: + await TwitchController().add_client(name) + create_channel(name) + except ChannelDoesNotExist: + logger.info(f"channel does not exist: {name}") + raise HTTPException(status_code=404, detail="channel does not exist") + except Exception as e: + logger.error(f"error creating the channel {e}") + raise HTTPException(status_code=500, detail="failed to create the channel") + + +@router.delete("/{name}", status_code=200) +def handle_delete_channel(name: str): + try: + delete_channel(name) + TwitchController().remove_client(name) + except ChannelDoesNotExist: + raise HTTPException(status_code=404, detail="channel was not found to delete") + except Exception as e: + logger.error(f"failed to delete the channel: {e}") + raise HTTPException(status_code=500, detail="failed to delete the channel") + + +@router.get("/", status_code=200) +def handle_get_all_channels(): + try: + channels = get_all_channels() + return {"channels": channels} + except Exception as e: + logger.error(f"failed to get all channels {e}") + raise HTTPException(status_code=500, detail="failed to get all channels") diff --git a/twitch-vod/src/twitch_vod/services/__init__.py b/twitch-vod/src/twitch_vod/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/twitch-vod/src/twitch_vod/services/bucket/__init__.py b/twitch-vod/src/twitch_vod/services/bucket/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/twitch-vod/src/twitch_vod/services/bucket/bucket.py b/twitch-vod/src/twitch_vod/services/bucket/bucket.py new file mode 100644 index 0000000..4839989 --- /dev/null +++ b/twitch-vod/src/twitch_vod/services/bucket/bucket.py @@ -0,0 +1,33 @@ +from boto3 import client +from botocore.exceptions import ClientError + +from twitch_vod.config import Config + + +class S3Bucket: + def __init__(self): + self._client = None + self.config = Config() + + def _get_client(self): + if self._client is None: + self._client = client( + "s3", + endpoint_url=self.config.S3_ENDPOINT_URL, + aws_access_key_id=self.config.S3_ACCESS_KEY, + aws_secret_access_key=self.config.S3_SECRET_KEY, + region_name=self.config.S3_REGION, + ) + return self._client + + def store_file(self, key: str, file_path: str, bucket_name: str): + try: + self._get_client().upload_file(file_path, bucket_name, key) + except ClientError as e: + raise RuntimeError(f"Failed to upload file: {e}") + + def get_file(self, key: str, file_path: str, bucket_name: str): + try: + self._get_client().download_file(bucket_name, key, file_path) + except ClientError as e: + raise RuntimeError(f"Failed to download file: {e}") diff --git a/twitch-vod/src/twitch_vod/services/db/__init__.py b/twitch-vod/src/twitch_vod/services/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/twitch-vod/src/twitch_vod/services/db/models/__init__.py b/twitch-vod/src/twitch_vod/services/db/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/twitch-vod/src/twitch_vod/services/db/models/channel.py b/twitch-vod/src/twitch_vod/services/db/models/channel.py new file mode 100644 index 0000000..429f7ff --- /dev/null +++ b/twitch-vod/src/twitch_vod/services/db/models/channel.py @@ -0,0 +1,5 @@ +from sqlmodel import Field, SQLModel + + +class Channel(SQLModel, table=True): + channel_name: str = Field(primary_key=True) diff --git a/twitch-vod/src/twitch_vod/services/db/models/vod.py b/twitch-vod/src/twitch_vod/services/db/models/vod.py new file mode 100644 index 0000000..e2767e6 --- /dev/null +++ b/twitch-vod/src/twitch_vod/services/db/models/vod.py @@ -0,0 +1,8 @@ +from uuid import uuid4 +from sqlmodel import UUID, Field, SQLModel + + +class VOD(SQLModel, table=True): + uuid: UUID = Field(default=uuid4(), primary_key=True) + vod_bucket_key: str + vod_bucket_message_key: str diff --git a/twitch-vod/src/twitch_vod/services/db/postgres.py b/twitch-vod/src/twitch_vod/services/db/postgres.py new file mode 100644 index 0000000..f55301e --- /dev/null +++ b/twitch-vod/src/twitch_vod/services/db/postgres.py @@ -0,0 +1,23 @@ +import logging + +from sqlmodel import create_engine + +from twitch_vod.config import Config + + +logger = logging.getLogger(__name__) +config = Config() + + +def postgres_engine(): + try: + connection_string = ( + f"postgresql://{config.POSTGRES_USER}:{config.POSTGRES_PASSWORD}" + f"@{config.POSTGRES_HOST}:{config.POSTGRES_PORT}/{config.POSTGRES_DB}" + ) + engine = create_engine(connection_string) + logger.info(f"successfully created the postgres engine: {engine}") + return engine + except Exception as e: + logger.error(f"failed to create the postgres engine: {e}") + raise RuntimeError(e) diff --git a/twitch-vod/src/twitch_vod/services/db/repository/__init__.py b/twitch-vod/src/twitch_vod/services/db/repository/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/twitch-vod/src/twitch_vod/services/db/repository/channel.py b/twitch-vod/src/twitch_vod/services/db/repository/channel.py new file mode 100644 index 0000000..f120caf --- /dev/null +++ b/twitch-vod/src/twitch_vod/services/db/repository/channel.py @@ -0,0 +1,45 @@ +import logging + +from sqlmodel import Session, select +from twitch_vod.services.db.models.channel import Channel +from twitch_vod.services.db.postgres import postgres_engine +from twitch_vod.services.twitch.models.errors import ChannelDoesNotExist + +logger = logging.getLogger(__name__) + + +def create_channel(channel_name): + channel = Channel(channel_name=channel_name) + + session = Session(postgres_engine()) + + session.add(channel) + session.commit() + session.refresh(channel) + + return channel + + +def delete_channel(channel_name: str): + session = Session(postgres_engine()) + + statement = select(Channel).where(Channel.channel_name == channel_name) + result = session.exec(statement) + + try: + channel = result.one() + session.delete(channel) + session.commit() + except Exception as e: + logger.info("failed to find channel to delete") + ChannelDoesNotExist(e) + + +def get_all_channels(): + session = Session(postgres_engine()) + + statement = select(Channel) + result = session.exec(statement) + channels = result.all() + + return channels diff --git a/twitch-vod/src/twitch_vod/services/db/repository/vod.py b/twitch-vod/src/twitch_vod/services/db/repository/vod.py new file mode 100644 index 0000000..501f419 --- /dev/null +++ b/twitch-vod/src/twitch_vod/services/db/repository/vod.py @@ -0,0 +1,27 @@ +from uuid import UUID +from sqlmodel import Session, select +from twitch_vod.services.db.models.vod import VOD +from twitch_vod.services.db.postgres import postgres_engine + + +def create_vod(bucket_video_key: str, bucket_msg_key: str): + vod = VOD(vod_bucket_key=bucket_video_key, vod_bucket_message_key=bucket_msg_key) + + session = Session(postgres_engine()) + + session.add(vod) + session.commit() + session.refresh(vod) + + return vod + + +def delete_vod(uuid: UUID): + session = Session(postgres_engine()) + + statement = select(VOD).where(VOD.uuid == uuid) + result = session.exec(statement) + vod = result.one() + + session.delete(vod) + session.commit() diff --git a/twitch-vod/src/twitch_vod/services/twitch/controller.py b/twitch-vod/src/twitch_vod/services/twitch/controller.py new file mode 100644 index 0000000..a0e2061 --- /dev/null +++ b/twitch-vod/src/twitch_vod/services/twitch/controller.py @@ -0,0 +1,78 @@ +from typing import Dict + +from loguru import logger +from twitchAPI.twitch import Twitch + +from twitch_vod.config import Config +from twitch_vod.services.twitch.models.errors import ChannelDoesNotExist + +from .twitch import _TwitchClient + + +class TwitchController: + _instance = None + _lock = __import__("threading").Lock() + + def __new__(cls): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self) -> None: + if not hasattr(self, "_initialized"): + self._clients: Dict[str, _TwitchClient] = {} + self._config = Config() + self._initialized = True + self._twitch_api_client = None + + async def create(self): + self._twitch_api_client = await Twitch( + Config().TWITCH_CLIENT_ID, # type: ignore + Config().TWITCH_SECRET, # type: ignore + ) + return self + + async def add_client(self, channel_name: str): + if self._twitch_api_client is None: + raise RuntimeError( + "need to use the `create` method when creating the TwitchController" + ) + twitch_client = _TwitchClient(channel_name, self._twitch_api_client) + + if not await self.real_channel(channel_name): + raise ChannelDoesNotExist("twitch channel not found") + + try: + await twitch_client.connect() + self._clients[channel_name] = twitch_client + except Exception: + logger.error(f"failed to add client {twitch_client}") + + async def real_channel(self, channel_name: str) -> bool: + try: + users = await self._twitch_api_client.get_users(logins=[channel_name]) # type: ignore + return len(users) > 0 + except Exception: + return False + + def remove_client(self, channel_name: str): + twitch_client = self._clients.get(channel_name) + + if twitch_client is None: + logger.warning( + f"tried to removed twitch client with channel name {channel_name}, however, it does not exist" + ) + return + + try: + twitch_client.disconnect() + except Exception: + logger.error(f"failed to discconect client {twitch_client}") + + del self._clients[channel_name] + + def cleanup(self): + for _, twitch_client in self._clients.items(): + twitch_client.disconnect() diff --git a/twitch-vod/src/twitch_vod/services/twitch/models/__init__.py b/twitch-vod/src/twitch_vod/services/twitch/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/twitch-vod/src/twitch_vod/services/twitch/models/errors.py b/twitch-vod/src/twitch_vod/services/twitch/models/errors.py new file mode 100644 index 0000000..1034361 --- /dev/null +++ b/twitch-vod/src/twitch_vod/services/twitch/models/errors.py @@ -0,0 +1,16 @@ +class IRCError(Exception): + """Error when communication with an IRC fails""" + + pass + + +class UnexpectedError(Exception): + """Catch all errors for errors we did not expect""" + + pass + + +class ChannelDoesNotExist(Exception): + """Channel does not exist""" + + pass diff --git a/twitch-vod/src/twitch_vod/services/twitch/models/message.py b/twitch-vod/src/twitch_vod/services/twitch/models/message.py new file mode 100644 index 0000000..0723174 --- /dev/null +++ b/twitch-vod/src/twitch_vod/services/twitch/models/message.py @@ -0,0 +1,8 @@ +from pydantic import BaseModel +from datetime import datetime + + +class Message(BaseModel): + username: str + message: str + timestamp: datetime diff --git a/twitch-vod/src/twitch_vod/services/twitch/twitch.py b/twitch-vod/src/twitch_vod/services/twitch/twitch.py new file mode 100644 index 0000000..cdf77a8 --- /dev/null +++ b/twitch-vod/src/twitch_vod/services/twitch/twitch.py @@ -0,0 +1,223 @@ +import socket +import time +import threading +import subprocess +from pathlib import Path +from loguru import logger + +from collections import deque +from typing import Optional +from datetime import datetime + +from twitchAPI.twitch import Twitch +from twitchAPI.oauth import UserAuthenticator +from twitchAPI.chat import Chat, ChatEvent, ChatMessage, EventData + +from twitchAPI.type import AuthScope + +from .models.errors import IRCError, UnexpectedError + + +class _TwitchClient: + async def __init__(self, channel: str, twitch_api: Twitch) -> None: + self._twitch_api = twitch_api + self._chat: Optional[Chat] = None + self._socket: Optional[socket.socket] = None + self._message_timestamps: deque[float] = deque() + self._start_time = datetime.now() + self._total_messages = 0 + self._connected = False + self._channel = channel + + self._failed_socket_attempts = 0 + + self._recording_thread: Optional[threading.Thread] = None + self._is_recording = False + self._should_stop_recording = False + self._current_vod_path: Optional[str] = None + self._recording_process: Optional[subprocess.Popen] = None + self._streamlink_process: Optional[subprocess.Popen] = None + + async def _get_user_access_token(self): + auth = UserAuthenticator(self._twitch_api, [AuthScope.CHAT_READ]) + token, refresh_token = await auth.authenticate() # type: ignore + await self._twitch_api.set_user_authentication( + token, [AuthScope.CHAT_READ], refresh_token + ) + + def get_connection_status(self): + return self._connected + + def get_channel(self): + return self._channel + + def get_total_messages(self): + return self._total_messages + + def get_start_time(self): + return self._start_time + + def start_recording_stream(self): + """Start recording the stream from twitch""" + if self._is_recording: + raise RuntimeError("Recording is already in progress") + + self._is_recording = True + self._should_stop_recording = False + + self._recording_thread = threading.Thread( + target=self._record_stream_thread, daemon=True + ) + self._recording_thread.start() + + logger.info(f"Started recording stream for channel {self._channel}") + + return True + + async def connect(self): + """High level connection function""" + logger.info(f"starting connection to Twitch channel {self._channel}") + try: + await self.irc_connect() + self.start_recording_stream() + logger.info( + f"successfully disconnected from twitch channel {self._channel}" + ) + except Exception as e: + logger.error(f"failed to connect to twitch channel {self._channel}") + raise Exception(e) + + def disconnect(self): + """High level disconnect function""" + logger.info(f"starting disconnecting from twitch channel {self._channel}") + try: + self.irc_disconnect() + self.stop_recording_stream() + logger.info( + f"successfully disconnected from twitch channel {self._channel}" + ) + except Exception as e: + logger.error(f"failed to disconnect from twitch channel {self._channel}") + raise Exception(e) + + def stop_recording_stream(self): + """Stops recording the stream from twitch""" + if not self._is_recording: + raise RuntimeError("No recording is in progress") + + self._should_stop_recording = True + + if self._streamlink_process: + logger.info("stopping stream link") + self._streamlink_process.kill() + self._streamlink_process = None + logger.info("success stopping stream link") + + if self._recording_process: + logger.info("stopping ffmpeg") + self._recording_process.kill() + self._recording_process = None + logger.info("success stopping ffmpeg") + + if self._recording_thread: + logger.info("waiting for recording thread to join back in") + self._recording_thread.join() + logger.info("threaded joined") + self._recording_thread = None + + self._is_recording = False + self._recording_thread = None + + logger.info(f"Stopped recording stream for channel {self._channel}") + + return True + + def _path(self): + return Path(f"vods/{self._channel}_{self._start_time}/") + + def _vod_path(self): + vods_dir = self._path() + vods_dir.mkdir(exist_ok=True) + + vod_filename = f"{self._channel}_{self._start_time}.mkv" + return vods_dir / vod_filename + + def _message_path(self): + msg_dir = self._path() + msg_dir.mkdir(exist_ok=True) + + return msg_dir / "messages.txt" + + def _record_stream_thread(self): + twitch_url = f"https://twitch.tv/{self._channel}" + self._current_vod_path = str(self._vod_path()) + + logger.info(f"Recording to file: {self._current_vod_path}") + + streamlink_cmd = [ + "streamlink", + twitch_url, + "best", + "--stdout", + "--twitch-supported-codecs", + "av1,h265", + ] + + ffmpeg_cmd = [ + "ffmpeg", + "-i", + "pipe:0", + "-vf", + "fps=24", + "-c:v", + "av1_nvenc", + "-preset", + "fast", + "-crf", + "23", + "-y", + self._current_vod_path, + ] + + self._streamlink_process = subprocess.Popen( + streamlink_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + self._recording_process = subprocess.Popen( + ffmpeg_cmd, stdin=self._streamlink_process.stdout, stderr=subprocess.PIPE + ) + + if self._streamlink_process.stdout: + self._streamlink_process.stdout.close() + + while True: + if self._should_stop_recording: + logger.info("Stop signal received, terminating recording") + break + + time.sleep(1) + + async def irc_connect(self): + self._chat = await Chat(self._twitch_api) + self._chat.register_event(ChatEvent.READY, self._chat_on_ready) + self._chat.register_event(ChatEvent.MESSAGE, self._chat_on_message) + self._chat.start() + + async def _chat_on_ready(self, ready_event: EventData): + await ready_event.chat.join_room(self._channel) + + async def _chat_on_message(self, msg: ChatMessage): + self._write_messages(f"{msg.user.name}: {msg.text}: {msg.sent_timestamp}") + + def irc_disconnect(self): + if self._chat is None: + return + self._chat.stop() + + def _write_messages(self, message: str): + """Takes a list of twitch irc messages""" + msg_path = self._message_path() + msg_path.touch(exist_ok=True) + + with open(msg_path, "a") as f: + f.write(f"{message}\n") diff --git a/twitch-vod/src/twitch_vod/tui.py b/twitch-vod/src/twitch_vod/tui.py new file mode 100644 index 0000000..02863a7 --- /dev/null +++ b/twitch-vod/src/twitch_vod/tui.py @@ -0,0 +1,109 @@ +from config import config_types, Config +from textual.app import App, ComposeResult +from textual.containers import Horizontal, Vertical +from textual.widgets import Input, Log, Button + + +import asyncio + +# this directly accesses the underlying Twitch client, since this +# is only intended to manually test the client +from services.twitch.twitch import _TwitchClient as TwitchClient + + +class TwitchTUI(App): + CSS = """ + .header { + height: 3; + dock: top; + } + .main-content { + height: 1fr; + } + #channel-input { + width: 1fr; + } + #connect-btn { + margin-left: 1; + } + #log { + height: 1fr; + border-top: solid $primary; + } + """ + + def __init__(self, config: Config) -> None: + super().__init__() + self.twitch_service = TwitchClient( + config.TWITCH_ACCESS_TOKEN, config.TWITCH_USERNAME, "" + ) + + def compose(self) -> ComposeResult: + with Vertical(classes="main-content"): + with Horizontal(classes="header"): + yield Input(placeholder="Enter channel name...", id="channel-input") + yield Button("Connect", id="connect-btn") + yield Button("Disconnect", id="disconnect-btn") + yield Log(id="log") + + async def update(self): + while self.twitch_service.get_connection_status(): + messages = self.twitch_service.read_messages() + for m in messages: + self.add_log(m) + await asyncio.sleep(1) + self.add_log("worker completed") + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == "connect-btn": + channel_input = self.query_one("#channel-input", Input) + channel = channel_input.value.strip() + if channel: + self.log(f"Connecting to channel: {channel}") + self.start(channel) + else: + self.log("Please enter a channel name", "warning") + elif event.button.id == "disconnect-btn": + self.log("Disconnection from channel") + self.stop() + + def on_input_submitted(self, event: Input.Submitted) -> None: + if event.input.id == "channel-input": + channel = event.input.value.strip() + if channel: + self.log(f"Connecting to channel: {channel}") + event.input.value = "" + else: + self.log("Please enter a channel name", "warning") + + def start(self, channel: str): + self.add_log("Connecting...") + try: + self.twitch_service._channel = channel + self.add_log("starting irc connection") + self.twitch_service.irc_connect() + self.add_log("successfully connected") + + self.add_log("starting stream recording") + self.twitch_service.start_recording_stream() + self.add_log("successfully connected") + + self.run_worker(self.update) + except Exception as e: + self.add_log(f"failed to connect: {e}") + self.stop() + + def stop(self): + try: + self.twitch_service.irc_disconnect() + self.twitch_service.stop_recording_stream() + except Exception: + pass + + def add_log(self, message: str, level: str = "info") -> None: + log_widget = self.query_one("#log", Log) + log_widget.write_line(f"[{level.upper()}] {message}") + + +if __name__ == "__main__": + TwitchTUI(config_types["development"]).run() diff --git a/twitch-vod/tests/__init__.py b/twitch-vod/tests/__init__.py new file mode 100644 index 0000000..e69de29