This commit is contained in:
BotoX 2021-05-20 00:31:31 +02:00
parent 30bb08712a
commit 90ea2174db
13 changed files with 10991 additions and 73 deletions

4
app.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import gevent.monkey # noqa isort:skip #import gevent.monkey # noqa isort:skip
gevent.monkey.patch_all() # noqa isort:skip #gevent.monkey.patch_all() # noqa isort:skip
from demweb.app import create_app from demweb.app import create_app

34
demopus.py Normal file
View File

@ -0,0 +1,34 @@
import struct
import sys
with open(sys.argv[1], 'rb') as fp:
data = fp.read()
ofs = 16
counter = 0
while ofs < len(data):
header = struct.unpack('B', data[ofs:ofs+1])[0]
ofs += 1
print(f'#{ofs} Header: {header}')
if header == 0x01:
srate = struct.unpack('I', data[ofs:ofs+4])[0]
ofs += 4
samples = struct.unpack('Q', data[ofs:ofs+8])[0]
ofs += 8
print(f'srate: {srate} | samples: {samples}\n')
elif header == 0x02:
dlen = struct.unpack('Q', data[ofs:ofs+8])[0]
ofs += 8
with open(f'{counter}.opus', 'wb') as fp:
fp.write(data[ofs:ofs+dlen])
counter += 1
ofs += dlen
print(f'opus len {dlen}')
elif header == 0x03:
silence = struct.unpack('Q', data[ofs:ofs+8])[0]
ofs += 8
print(f'silence {silence}')
elif header == 0x04:
break

View File

@ -10,10 +10,19 @@ def register_extensions(app):
db.init_app(app) db.init_app(app)
def register_jinja2_filters(app):
import orjson
with app.app_context():
@app.template_filter('to_json')
def to_json(value):
return orjson.dumps(value).decode('utf-8')
def register_blueprints(app): def register_blueprints(app):
with app.app_context(): with app.app_context():
# Register blueprints # Register blueprints
pass from .views import bp as views_bp
app.register_blueprint(views_bp, url_prefix='/')
def setup_logging(app): def setup_logging(app):
@ -77,6 +86,7 @@ def create_app(test_config=None):
app.config.update(test_config) app.config.update(test_config)
register_extensions(app) register_extensions(app)
register_jinja2_filters(app)
register_blueprints(app) register_blueprints(app)
register_shell_context(app) register_shell_context(app)
setup_logging(app) setup_logging(app)

View File

@ -1,8 +1,10 @@
from flask_sqlalchemy import SQLAlchemy from flask_sqlalchemy import SQLAlchemy
from dictalchemy import make_class_dictable
import config import config
db = SQLAlchemy() db = SQLAlchemy()
make_class_dictable(db.Model)
default_config = dict() default_config = dict()
for key in dir(config): for key in dir(config):

View File

@ -62,13 +62,13 @@ class PlayerSession(db.Model):
deaths = db.Column(db.Integer) deaths = db.Column(db.Integer)
kills = db.Column(db.Integer) kills = db.Column(db.Integer)
voicetime = db.Column(db.Float) voicetime = db.Column(db.Float)
voice_chunks = db.Column(db.JSON(65535))
class Chat(db.Model): class Chat(db.Model):
id = db.Column(db.Integer, primary_key=True) id = db.Column(db.Integer, primary_key=True)
player_guid = db.Column(db.String(32), db.ForeignKey('player.guid')) player_guid = db.Column(db.String(32), db.ForeignKey('player.guid'))
session_id = db.Column(db.Integer, db.ForeignKey('session.id')) session_id = db.Column(db.Integer, db.ForeignKey('session.id'))
tick = db.Column(db.Integer)
time = db.Column(db.DateTime, index=True) time = db.Column(db.DateTime, index=True)
name = db.Column(db.String(255)) name = db.Column(db.String(255))
chat = db.Column(db.String(1024)) chat = db.Column(db.String(1024))
@ -78,6 +78,7 @@ class Event(db.Model):
id = db.Column(db.Integer, primary_key=True) id = db.Column(db.Integer, primary_key=True)
player_guid = db.Column(db.String(32), db.ForeignKey('player.guid')) player_guid = db.Column(db.String(32), db.ForeignKey('player.guid'))
session_id = db.Column(db.Integer, db.ForeignKey('session.id')) session_id = db.Column(db.Integer, db.ForeignKey('session.id'))
tick = db.Column(db.Integer)
time = db.Column(db.DateTime, index=True) time = db.Column(db.DateTime, index=True)
event = db.Column(db.String(32), index=True) event = db.Column(db.String(32), index=True)
data = db.Column(db.JSON(1024)) data = db.Column(db.JSON(1024))

1
demweb/static/css-ze-parsed Symbolic link
View File

@ -0,0 +1 @@
/home/david/Projects/demboyz/premake/gmake

View File

@ -0,0 +1,40 @@
.btn-group-xs > .btn, .btn-xs { padding: .25rem .4rem; font-size: .875rem; line-height: .5; border-radius: .2rem; }
.playlist { margin: 2em 0; }
.playlist .playlist-time-scale { height: 30px; }
.playlist .playlist-tracks { background: #e0eff1; }
.playlist .channel { background: grey; }
.playlist .channel-progress { background: orange; }
.playlist .cursor { background: black; }
.playlist .wp-fade { background-color: rgba(0, 0, 0, 0.1); }
.playlist .state-cursor, .playlist .state-select { cursor: text; }
.playlist .state-fadein { cursor: w-resize; }
.playlist .state-fadeout { cursor: e-resize; }
.playlist .state-shift { cursor: ew-resize; }
.playlist .selection.point { background: red; }
.playlist .selection.segment { background: rgba(0, 0, 0, 0.1); }
.playlist .channel-wrapper.silent .channel { opacity: 0.3; }
.playlist .controls { background: white; text-align: center; border: 1px solid black; border-radius: 0.2rem; }
.playlist .controls .track-header { overflow: hidden; color: black; height: 26px; display: flex; align-items: center; justify-content: space-between; padding: 0 0.2rem; font-size: 0.65rem; margin-bottom: -10px; }
.playlist .controls .track-header button { width: 20px; height: 20px; display: flex; align-items: center; justify-content: center; }
.playlist .controls input[type="range"] { display: inline-block; width: 90%; }
.playlist .controls .user-info { word-break: break-word; color: black; align-items: center; justify-content: space-between; padding: 0 0.2rem; font-size: 0.65rem; margin-top: -5px; }

129
demweb/static/js/main.js Normal file
View File

@ -0,0 +1,129 @@
var playlist = WaveformPlaylist.init({
container: document.getElementById("playlist"),
timescale: true,
state: 'cursor',
samplesPerPixel: 16384,
zoomLevels: [2048, 4096, 8192, 16384],
controls: {
show: true,
width: 150,
widgets: {
stereoPan: false,
}
},
waveHeight: 96,
});
function updateTrackInfo(guid, info) {
var tracks = $(".playlist-tracks").children();
for (var i = 0; i < tracks.length; i++) {
var track = tracks[i].firstChild;
var trackGuid = track.firstChild.getElementsByTagName('span')[0].innerText;
var userInfoElem = track.lastChild;
if (userInfoElem.className != 'user-info') {
userInfoElem = document.createElement('label');
userInfoElem.className = 'user-info';
track.appendChild(userInfoElem);
}
if (guid === null || trackGuid == guid) {
userInfoElem.innerText = info;
}
}
}
var ee = playlist.getEventEmitter();
var $container = $("body");
var $time = $container.find(".audio-pos");
var $time_ = $container.find(".audio-pos-2");
var audioPos = 0;
function clockFormat(seconds, decimals) {
var hours,
minutes,
secs,
result;
hours = parseInt(seconds / 3600, 10) % 24;
minutes = parseInt(seconds / 60, 10) % 60;
secs = seconds % 60;
secs = secs.toFixed(decimals);
result = (hours < 10 ? "0" + hours : hours) + ":" + (minutes < 10 ? "0" + minutes : minutes) + ":" + (secs < 10 ? "0" + secs : secs);
return result;
}
function updateTime(time) {
$time.html(clockFormat(time, 3));
audioPos = time;
var tick = time / g_session.tickinterval;
var silenceTicks = 0;
for (var i = 0; i < g_session.silence_chunks.length; i++) {
var chunk = g_session.silence_chunks[i];
if (tick > chunk[0]) {
silenceTicks += chunk[1];
} else {
break;
}
}
var tickedTime = (tick + silenceTicks) * g_session.tickinterval;
$time_.html(clockFormat(tickedTime, 3));
}
updateTime(audioPos);
$container.on("click", ".btn-play", function() {
ee.emit("play");
});
$container.on("click", ".btn-pause", function() {
isLooping = false;
ee.emit("pause");
});
$container.on("click", ".btn-stop", function() {
isLooping = false;
ee.emit("stop");
});
$container.on("click", ".btn-rewind", function() {
isLooping = false;
ee.emit("rewind");
});
$container.on("click", ".btn-fast-forward", function() {
isLooping = false;
ee.emit("fastforward");
});
//zoom buttons
$container.on("click", ".btn-zoom-in", function() {
ee.emit("zoomin");
});
$container.on("click", ".btn-zoom-out", function() {
ee.emit("zoomout");
});
$container.on("input change", ".master-gain", function(e){
ee.emit("mastervolumechange", e.target.value);
});
$container.find(".master-gain").change();
$container.on("change", ".automatic-scroll", function(e){
ee.emit("automaticscroll", $(e.target).is(':checked'));
});
$container.find(".automatic-scroll").change();
ee.on("timeupdate", updateTime);
function onFinishedLoading() {
updateTrackInfo(null, "name");
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,166 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>
{{ session.demoname }}
</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.1/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-+0n0xVW2eSR5OomGNYDnhzAbDsOXxcvSN1TPprVMTNDbiYZCxYbOOl7+AMvyTG2x" crossorigin="anonymous">
<link rel="stylesheet" href="/static/css/main.css">
<script src="https://kit.fontawesome.com/ef69927139.js" crossorigin="anonymous"></script>
</head>
<body>
<nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4">
<div class="container-fluid">
<a class="navbar-brand" href="#">Top navbar</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse" id="navbarCollapse">
<ul class="navbar-nav me-auto mb-2 mb-md-0">
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="#">Home</a>
</li>
<li class="nav-item">
<a class="nav-link" href="#">Link</a>
</li>
<li class="nav-item">
<a class="nav-link disabled" href="#" tabindex="-1" aria-disabled="true">Disabled</a>
</li>
</ul>
<form class="d-flex">
<input class="form-control me-2" type="search" placeholder="Search" aria-label="Search">
<button class="btn btn-outline-success" type="submit">Search</button>
</form>
</div>
</div>
</nav>
<main class="container-fluid">
<div class="btn-toolbar" role="toolbar">
<div class="btn-group me-2" role="group">
<button type="button" class="btn-pause btn btn-outline-warning" title="Pause">
<i class="fas fa-pause"></i>
</button>
<button type="button" class="btn-play btn btn-outline-success" title="Play">
<i class="fas fa-play"></i>
</button>
<button type="button" class="btn-stop btn btn-outline-danger" title="Stop">
<i class="fas fa-stop"></i>
</button>
<button
type="button"
class="btn-rewind btn btn-outline-success"
title="Rewind"
>
<i class="fas fa-fast-backward"></i>
</button>
<button
type="button"
class="btn-fast-forward btn btn-outline-success"
title="Fast forward"
>
<i class="fas fa-fast-forward"></i>
</button>
</div>
<div class="btn-group me-2" role="group">
<button type="button" title="Zoom in" class="btn-zoom-in btn btn-outline-dark">
<i class="fas fa-search-plus" aria-hidden="true"></i>
</button>
<button type="button" title="Zoom out" class="btn-zoom-out btn btn-outline-dark">
<i class="fas fa-search-minus" aria-hidden="true"></i>
</button>
</div>
<div class="btn-group me-2">
<div style="margin: 6px">
<input
type="range"
min="0"
max="100"
value="50"
class="master-gain form-range mw-50"
id="master-gain"
/>
</div>
<div style="margin: 6px">
<span class="audio-pos" aria-label="Audio position">00:00:00.0</span>
</div>
<div style="margin: 6px">
<span class="audio-pos-2" aria-label="Audio position">00:00:00.0</span>
</div>
<div class="form-check form-switch">
<input class="form-check-input automatic-scroll" type="checkbox" id="automatic_scroll" checked>
<label class="form-check-label" for="automatic_scroll">Autoscroll</label>
</div>
</div>
</div>
<div id="playlist">
</div>
</main>
<script
src="https://code.jquery.com/jquery-3.6.0.min.js"
integrity="sha256-/xUj+3OJU5yExlq6GSYGSHk7tPXikynS7ogEvDej/m4="
crossorigin="anonymous"
></script>
<script
src="/static/js/waveform-playlist.var.js?v=4.0.1"
></script>
<script type="text/javascript">
var g_session = {{ session.asdict() | to_json | safe }};
var g_player_sessions = {
{%- for guid, psess in player_sessions.items() %}
"{{ guid }}": {{ psess.asdict(exclude=['player_guid', 'session_id']) | to_json | safe }},
{%- endfor %}
};
var g_chats = [
{%- for chat in chats %}
{{ chat.asdict(exclude=['id', 'session_id']) | to_json | safe }},
{%- endfor %}
];
var g_events = [
{%- for event in events %}
{{ event.asdict(exclude=['id', 'session_id']) | to_json | safe }},
{%- endfor %}
];
</script>
<script
src="/static/js/main.js"
></script>
<script type="text/javascript">
playlist.load([
{%- for guid, psess in player_sessions.items() %}
{%- if psess.voicetime > 0 %}
{src: "/static/css-ze-parsed/{{ session.demoname }}/voice/{{ guid }}.demopus", name: "{{ guid }}"},
{%- endif -%}
{% endfor %}
]).then(function() {
onFinishedLoading();
});
</script>
</body>
</html>

42
demweb/views.py Normal file
View File

@ -0,0 +1,42 @@
from flask import Blueprint, render_template
from demweb.extensions import db
from demweb.models import *
bp = Blueprint('views', __name__)
@bp.route('')
def home():
return 'hello world'
@bp.route('/session/<int:session_id>')
def session(session_id):
session = Session.query.filter_by(id=session_id).one()
if not session:
return '404 Not Found', 404
player_sessions_ = PlayerSession.query.filter_by(session_id=session_id).all()
player_sessions = dict()
for psess in player_sessions_:
player_sessions[psess.player_guid] = psess
del player_sessions_
players_ = Player.query.filter(Player.guid.in_(player_sessions.keys()))
players = dict()
for player in players_:
players[player.guid] = player
del players_
chats = Chat.query.filter_by(session_id=session_id).all()
chats.sort(key=lambda x: x.tick)
events = Event.query.filter_by(session_id=session_id).all()
events.sort(key=lambda x: x.tick)
return render_template('session.html',
session=session,
player_sessions=player_sessions,
players=players,
chats=chats,
events=events
)

View File

@ -7,7 +7,6 @@ import multiprocessing
import signal import signal
import re import re
import traceback import traceback
import subprocess
from datetime import datetime, timedelta from datetime import datetime, timedelta
from sqlalchemy import exc, func from sqlalchemy import exc, func
from sqlalchemy.sql.expression import bindparam from sqlalchemy.sql.expression import bindparam
@ -45,61 +44,6 @@ def remove_chatcolors(message):
i += 1 i += 1
return output return output
silence_start_re = re.compile(' silence_start: (?P<start>[0-9]+(\.?[0-9]*))$')
silence_end_re = re.compile(' silence_end: (?P<end>[0-9]+(\.?[0-9]*)) ')
total_duration_re = re.compile('size=[^ ]+ time=(?P<hours>[0-9]{2}):(?P<minutes>[0-9]{2}):(?P<seconds>[0-9\.]{5}) bitrate=')
def get_chunk_times(path):
proc = subprocess.Popen([
'ffmpeg',
'-i', path,
'-af', 'silencedetect=n=-80dB:d=0.1',
'-f', 'null',
'-'],
stderr=subprocess.PIPE,
stdout=subprocess.DEVNULL
)
output = proc.communicate()[1].decode('utf-8')
lines = output.splitlines()
# Chunks start when silence ends, and chunks end when silence starts.
chunk_starts = []
chunk_ends = []
for line in lines:
silence_start_match = silence_start_re.search(line)
silence_end_match = silence_end_re.search(line)
total_duration_match = total_duration_re.search(line)
if silence_start_match:
start = float(silence_start_match.group('start'))
if start == 0.:
# Ignore initial silence.
continue
chunk_ends.append(start)
if len(chunk_starts) == 0:
# Started with non-silence.
chunk_starts.append(0.)
elif silence_end_match:
chunk_starts.append(float(silence_end_match.group('end')))
elif total_duration_match:
hours = int(total_duration_match.group('hours'))
minutes = int(total_duration_match.group('minutes'))
seconds = float(total_duration_match.group('seconds'))
end_time = hours * 3600 + minutes * 60 + seconds
if len(chunk_starts) == 0:
# No silence found.
chunk_starts.append(0)
if len(chunk_starts) > len(chunk_ends):
if abs(chunk_starts[-1] - end_time) < 0.1:
# Last chunk starts at very the end? nah.
del chunk_starts[-1]
else:
# Finished with non-silence.
chunk_ends.append(end_time)
return list(zip(chunk_starts, chunk_ends))
def parse_demo(path): def parse_demo(path):
jmodified = False jmodified = False
@ -243,13 +187,6 @@ def parse_demo(path):
db.session.rollback() db.session.rollback()
print('deadlock 5') print('deadlock 5')
# calculate player voice chunks/timing if possible and not exists
if obj['voicetime'] > 0 and 'voice_chunks' not in obj:
voicefile = DEMO_PATH_PREFIX + path + '/voice/' + guid + '.opus'
if os.path.isfile(voicefile):
obj['voice_chunks'] = get_chunk_times(voicefile)
jmodified = True
# insert new player session # insert new player session
player_session = models.PlayerSession( player_session = models.PlayerSession(
player_guid=guid, player_guid=guid,
@ -258,8 +195,7 @@ def parse_demo(path):
chats=obj['chats'], chats=obj['chats'],
deaths=obj['deaths'], deaths=obj['deaths'],
kills=obj['kills'], kills=obj['kills'],
voicetime=obj['voicetime'], voicetime=obj['voicetime']
voice_chunks=obj['voice_chunks'] if 'voice_chunks' in obj else None
) )
db.session.add(player_session) db.session.add(player_session)
db.session.commit() db.session.commit()
@ -285,12 +221,18 @@ def parse_demo(path):
elif obj['winner'] == 3: elif obj['winner'] == 3:
session.ct_wins += 1 session.ct_wins += 1
# remove redundant info
eventdata = obj.copy()
del eventdata['tick']
del eventdata['event']
event = models.Event( event = models.Event(
player_guid=guid, player_guid=guid,
session_id=session.id, session_id=session.id,
time=starttime + timedelta(seconds=obj['tick'] / tickinterval), tick=obj['tick'],
time=starttime + timedelta(seconds=obj['tick'] * tickinterval),
event=obj['event'], event=obj['event'],
data=obj data=eventdata
) )
db.session.add(event) db.session.add(event)
db.session.commit() db.session.commit()
@ -318,7 +260,8 @@ def parse_demo(path):
chat = models.Chat( chat = models.Chat(
player_guid=obj['steamid'], player_guid=obj['steamid'],
session_id=session.id, session_id=session.id,
time=starttime + timedelta(seconds=obj['tick'] / tickinterval), tick=obj['tick'],
time=starttime + timedelta(seconds=obj['tick'] * tickinterval),
name=nick, name=nick,
chat=msg chat=msg
) )
@ -361,7 +304,7 @@ if __name__ == '__main__':
with open('done.txt', 'r') as fp: with open('done.txt', 'r') as fp:
done = set(fp.read().splitlines()) done = set(fp.read().splitlines())
if True: if False:
done = set() done = set()
app = create_app() app = create_app()
with app.app_context(): with app.app_context():

View File

@ -1,4 +1,5 @@
click==8.0.0 click==8.0.0
dictalchemy==0.1.2.7
Flask==2.0.0 Flask==2.0.0
Flask-SQLAlchemy==2.5.1 Flask-SQLAlchemy==2.5.1
gevent==21.1.2 gevent==21.1.2