This commit is contained in:
BotoX 2021-05-20 00:31:31 +02:00
parent 30bb08712a
commit 90ea2174db
13 changed files with 10991 additions and 73 deletions

4
app.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3
import gevent.monkey # noqa isort:skip
gevent.monkey.patch_all() # noqa isort:skip
#import gevent.monkey # noqa isort:skip
#gevent.monkey.patch_all() # noqa isort:skip
from demweb.app import create_app

34
demopus.py Normal file
View File

@ -0,0 +1,34 @@
import struct
import sys
with open(sys.argv[1], 'rb') as fp:
data = fp.read()
ofs = 16
counter = 0
while ofs < len(data):
header = struct.unpack('B', data[ofs:ofs+1])[0]
ofs += 1
print(f'#{ofs} Header: {header}')
if header == 0x01:
srate = struct.unpack('I', data[ofs:ofs+4])[0]
ofs += 4
samples = struct.unpack('Q', data[ofs:ofs+8])[0]
ofs += 8
print(f'srate: {srate} | samples: {samples}\n')
elif header == 0x02:
dlen = struct.unpack('Q', data[ofs:ofs+8])[0]
ofs += 8
with open(f'{counter}.opus', 'wb') as fp:
fp.write(data[ofs:ofs+dlen])
counter += 1
ofs += dlen
print(f'opus len {dlen}')
elif header == 0x03:
silence = struct.unpack('Q', data[ofs:ofs+8])[0]
ofs += 8
print(f'silence {silence}')
elif header == 0x04:
break

View File

@ -10,10 +10,19 @@ def register_extensions(app):
db.init_app(app)
def register_jinja2_filters(app):
import orjson
with app.app_context():
@app.template_filter('to_json')
def to_json(value):
return orjson.dumps(value).decode('utf-8')
def register_blueprints(app):
with app.app_context():
# Register blueprints
pass
from .views import bp as views_bp
app.register_blueprint(views_bp, url_prefix='/')
def setup_logging(app):
@ -77,6 +86,7 @@ def create_app(test_config=None):
app.config.update(test_config)
register_extensions(app)
register_jinja2_filters(app)
register_blueprints(app)
register_shell_context(app)
setup_logging(app)

View File

@ -1,8 +1,10 @@
from flask_sqlalchemy import SQLAlchemy
from dictalchemy import make_class_dictable
import config
db = SQLAlchemy()
make_class_dictable(db.Model)
default_config = dict()
for key in dir(config):

View File

@ -62,13 +62,13 @@ class PlayerSession(db.Model):
deaths = db.Column(db.Integer)
kills = db.Column(db.Integer)
voicetime = db.Column(db.Float)
voice_chunks = db.Column(db.JSON(65535))
class Chat(db.Model):
id = db.Column(db.Integer, primary_key=True)
player_guid = db.Column(db.String(32), db.ForeignKey('player.guid'))
session_id = db.Column(db.Integer, db.ForeignKey('session.id'))
tick = db.Column(db.Integer)
time = db.Column(db.DateTime, index=True)
name = db.Column(db.String(255))
chat = db.Column(db.String(1024))
@ -78,6 +78,7 @@ class Event(db.Model):
id = db.Column(db.Integer, primary_key=True)
player_guid = db.Column(db.String(32), db.ForeignKey('player.guid'))
session_id = db.Column(db.Integer, db.ForeignKey('session.id'))
tick = db.Column(db.Integer)
time = db.Column(db.DateTime, index=True)
event = db.Column(db.String(32), index=True)
data = db.Column(db.JSON(1024))

1
demweb/static/css-ze-parsed Symbolic link
View File

@ -0,0 +1 @@
/home/david/Projects/demboyz/premake/gmake

View File

@ -0,0 +1,40 @@
.btn-group-xs > .btn, .btn-xs { padding: .25rem .4rem; font-size: .875rem; line-height: .5; border-radius: .2rem; }
.playlist { margin: 2em 0; }
.playlist .playlist-time-scale { height: 30px; }
.playlist .playlist-tracks { background: #e0eff1; }
.playlist .channel { background: grey; }
.playlist .channel-progress { background: orange; }
.playlist .cursor { background: black; }
.playlist .wp-fade { background-color: rgba(0, 0, 0, 0.1); }
.playlist .state-cursor, .playlist .state-select { cursor: text; }
.playlist .state-fadein { cursor: w-resize; }
.playlist .state-fadeout { cursor: e-resize; }
.playlist .state-shift { cursor: ew-resize; }
.playlist .selection.point { background: red; }
.playlist .selection.segment { background: rgba(0, 0, 0, 0.1); }
.playlist .channel-wrapper.silent .channel { opacity: 0.3; }
.playlist .controls { background: white; text-align: center; border: 1px solid black; border-radius: 0.2rem; }
.playlist .controls .track-header { overflow: hidden; color: black; height: 26px; display: flex; align-items: center; justify-content: space-between; padding: 0 0.2rem; font-size: 0.65rem; margin-bottom: -10px; }
.playlist .controls .track-header button { width: 20px; height: 20px; display: flex; align-items: center; justify-content: center; }
.playlist .controls input[type="range"] { display: inline-block; width: 90%; }
.playlist .controls .user-info { word-break: break-word; color: black; align-items: center; justify-content: space-between; padding: 0 0.2rem; font-size: 0.65rem; margin-top: -5px; }

129
demweb/static/js/main.js Normal file
View File

@ -0,0 +1,129 @@
var playlist = WaveformPlaylist.init({
container: document.getElementById("playlist"),
timescale: true,
state: 'cursor',
samplesPerPixel: 16384,
zoomLevels: [2048, 4096, 8192, 16384],
controls: {
show: true,
width: 150,
widgets: {
stereoPan: false,
}
},
waveHeight: 96,
});
function updateTrackInfo(guid, info) {
var tracks = $(".playlist-tracks").children();
for (var i = 0; i < tracks.length; i++) {
var track = tracks[i].firstChild;
var trackGuid = track.firstChild.getElementsByTagName('span')[0].innerText;
var userInfoElem = track.lastChild;
if (userInfoElem.className != 'user-info') {
userInfoElem = document.createElement('label');
userInfoElem.className = 'user-info';
track.appendChild(userInfoElem);
}
if (guid === null || trackGuid == guid) {
userInfoElem.innerText = info;
}
}
}
var ee = playlist.getEventEmitter();
var $container = $("body");
var $time = $container.find(".audio-pos");
var $time_ = $container.find(".audio-pos-2");
var audioPos = 0;
function clockFormat(seconds, decimals) {
var hours,
minutes,
secs,
result;
hours = parseInt(seconds / 3600, 10) % 24;
minutes = parseInt(seconds / 60, 10) % 60;
secs = seconds % 60;
secs = secs.toFixed(decimals);
result = (hours < 10 ? "0" + hours : hours) + ":" + (minutes < 10 ? "0" + minutes : minutes) + ":" + (secs < 10 ? "0" + secs : secs);
return result;
}
function updateTime(time) {
$time.html(clockFormat(time, 3));
audioPos = time;
var tick = time / g_session.tickinterval;
var silenceTicks = 0;
for (var i = 0; i < g_session.silence_chunks.length; i++) {
var chunk = g_session.silence_chunks[i];
if (tick > chunk[0]) {
silenceTicks += chunk[1];
} else {
break;
}
}
var tickedTime = (tick + silenceTicks) * g_session.tickinterval;
$time_.html(clockFormat(tickedTime, 3));
}
updateTime(audioPos);
$container.on("click", ".btn-play", function() {
ee.emit("play");
});
$container.on("click", ".btn-pause", function() {
isLooping = false;
ee.emit("pause");
});
$container.on("click", ".btn-stop", function() {
isLooping = false;
ee.emit("stop");
});
$container.on("click", ".btn-rewind", function() {
isLooping = false;
ee.emit("rewind");
});
$container.on("click", ".btn-fast-forward", function() {
isLooping = false;
ee.emit("fastforward");
});
//zoom buttons
$container.on("click", ".btn-zoom-in", function() {
ee.emit("zoomin");
});
$container.on("click", ".btn-zoom-out", function() {
ee.emit("zoomout");
});
$container.on("input change", ".master-gain", function(e){
ee.emit("mastervolumechange", e.target.value);
});
$container.find(".master-gain").change();
$container.on("change", ".automatic-scroll", function(e){
ee.emit("automaticscroll", $(e.target).is(':checked'));
});
$container.find(".automatic-scroll").change();
ee.on("timeupdate", updateTime);
function onFinishedLoading() {
updateTrackInfo(null, "name");
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,166 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>
{{ session.demoname }}
</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.1/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-+0n0xVW2eSR5OomGNYDnhzAbDsOXxcvSN1TPprVMTNDbiYZCxYbOOl7+AMvyTG2x" crossorigin="anonymous">
<link rel="stylesheet" href="/static/css/main.css">
<script src="https://kit.fontawesome.com/ef69927139.js" crossorigin="anonymous"></script>
</head>
<body>
<nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4">
<div class="container-fluid">
<a class="navbar-brand" href="#">Top navbar</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse" id="navbarCollapse">
<ul class="navbar-nav me-auto mb-2 mb-md-0">
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="#">Home</a>
</li>
<li class="nav-item">
<a class="nav-link" href="#">Link</a>
</li>
<li class="nav-item">
<a class="nav-link disabled" href="#" tabindex="-1" aria-disabled="true">Disabled</a>
</li>
</ul>
<form class="d-flex">
<input class="form-control me-2" type="search" placeholder="Search" aria-label="Search">
<button class="btn btn-outline-success" type="submit">Search</button>
</form>
</div>
</div>
</nav>
<main class="container-fluid">
<div class="btn-toolbar" role="toolbar">
<div class="btn-group me-2" role="group">
<button type="button" class="btn-pause btn btn-outline-warning" title="Pause">
<i class="fas fa-pause"></i>
</button>
<button type="button" class="btn-play btn btn-outline-success" title="Play">
<i class="fas fa-play"></i>
</button>
<button type="button" class="btn-stop btn btn-outline-danger" title="Stop">
<i class="fas fa-stop"></i>
</button>
<button
type="button"
class="btn-rewind btn btn-outline-success"
title="Rewind"
>
<i class="fas fa-fast-backward"></i>
</button>
<button
type="button"
class="btn-fast-forward btn btn-outline-success"
title="Fast forward"
>
<i class="fas fa-fast-forward"></i>
</button>
</div>
<div class="btn-group me-2" role="group">
<button type="button" title="Zoom in" class="btn-zoom-in btn btn-outline-dark">
<i class="fas fa-search-plus" aria-hidden="true"></i>
</button>
<button type="button" title="Zoom out" class="btn-zoom-out btn btn-outline-dark">
<i class="fas fa-search-minus" aria-hidden="true"></i>
</button>
</div>
<div class="btn-group me-2">
<div style="margin: 6px">
<input
type="range"
min="0"
max="100"
value="50"
class="master-gain form-range mw-50"
id="master-gain"
/>
</div>
<div style="margin: 6px">
<span class="audio-pos" aria-label="Audio position">00:00:00.0</span>
</div>
<div style="margin: 6px">
<span class="audio-pos-2" aria-label="Audio position">00:00:00.0</span>
</div>
<div class="form-check form-switch">
<input class="form-check-input automatic-scroll" type="checkbox" id="automatic_scroll" checked>
<label class="form-check-label" for="automatic_scroll">Autoscroll</label>
</div>
</div>
</div>
<div id="playlist">
</div>
</main>
<script
src="https://code.jquery.com/jquery-3.6.0.min.js"
integrity="sha256-/xUj+3OJU5yExlq6GSYGSHk7tPXikynS7ogEvDej/m4="
crossorigin="anonymous"
></script>
<script
src="/static/js/waveform-playlist.var.js?v=4.0.1"
></script>
<script type="text/javascript">
var g_session = {{ session.asdict() | to_json | safe }};
var g_player_sessions = {
{%- for guid, psess in player_sessions.items() %}
"{{ guid }}": {{ psess.asdict(exclude=['player_guid', 'session_id']) | to_json | safe }},
{%- endfor %}
};
var g_chats = [
{%- for chat in chats %}
{{ chat.asdict(exclude=['id', 'session_id']) | to_json | safe }},
{%- endfor %}
];
var g_events = [
{%- for event in events %}
{{ event.asdict(exclude=['id', 'session_id']) | to_json | safe }},
{%- endfor %}
];
</script>
<script
src="/static/js/main.js"
></script>
<script type="text/javascript">
playlist.load([
{%- for guid, psess in player_sessions.items() %}
{%- if psess.voicetime > 0 %}
{src: "/static/css-ze-parsed/{{ session.demoname }}/voice/{{ guid }}.demopus", name: "{{ guid }}"},
{%- endif -%}
{% endfor %}
]).then(function() {
onFinishedLoading();
});
</script>
</body>
</html>

42
demweb/views.py Normal file
View File

@ -0,0 +1,42 @@
from flask import Blueprint, render_template
from demweb.extensions import db
from demweb.models import *
bp = Blueprint('views', __name__)
@bp.route('')
def home():
return 'hello world'
@bp.route('/session/<int:session_id>')
def session(session_id):
session = Session.query.filter_by(id=session_id).one()
if not session:
return '404 Not Found', 404
player_sessions_ = PlayerSession.query.filter_by(session_id=session_id).all()
player_sessions = dict()
for psess in player_sessions_:
player_sessions[psess.player_guid] = psess
del player_sessions_
players_ = Player.query.filter(Player.guid.in_(player_sessions.keys()))
players = dict()
for player in players_:
players[player.guid] = player
del players_
chats = Chat.query.filter_by(session_id=session_id).all()
chats.sort(key=lambda x: x.tick)
events = Event.query.filter_by(session_id=session_id).all()
events.sort(key=lambda x: x.tick)
return render_template('session.html',
session=session,
player_sessions=player_sessions,
players=players,
chats=chats,
events=events
)

View File

@ -7,7 +7,6 @@ import multiprocessing
import signal
import re
import traceback
import subprocess
from datetime import datetime, timedelta
from sqlalchemy import exc, func
from sqlalchemy.sql.expression import bindparam
@ -45,61 +44,6 @@ def remove_chatcolors(message):
i += 1
return output
silence_start_re = re.compile(' silence_start: (?P<start>[0-9]+(\.?[0-9]*))$')
silence_end_re = re.compile(' silence_end: (?P<end>[0-9]+(\.?[0-9]*)) ')
total_duration_re = re.compile('size=[^ ]+ time=(?P<hours>[0-9]{2}):(?P<minutes>[0-9]{2}):(?P<seconds>[0-9\.]{5}) bitrate=')
def get_chunk_times(path):
proc = subprocess.Popen([
'ffmpeg',
'-i', path,
'-af', 'silencedetect=n=-80dB:d=0.1',
'-f', 'null',
'-'],
stderr=subprocess.PIPE,
stdout=subprocess.DEVNULL
)
output = proc.communicate()[1].decode('utf-8')
lines = output.splitlines()
# Chunks start when silence ends, and chunks end when silence starts.
chunk_starts = []
chunk_ends = []
for line in lines:
silence_start_match = silence_start_re.search(line)
silence_end_match = silence_end_re.search(line)
total_duration_match = total_duration_re.search(line)
if silence_start_match:
start = float(silence_start_match.group('start'))
if start == 0.:
# Ignore initial silence.
continue
chunk_ends.append(start)
if len(chunk_starts) == 0:
# Started with non-silence.
chunk_starts.append(0.)
elif silence_end_match:
chunk_starts.append(float(silence_end_match.group('end')))
elif total_duration_match:
hours = int(total_duration_match.group('hours'))
minutes = int(total_duration_match.group('minutes'))
seconds = float(total_duration_match.group('seconds'))
end_time = hours * 3600 + minutes * 60 + seconds
if len(chunk_starts) == 0:
# No silence found.
chunk_starts.append(0)
if len(chunk_starts) > len(chunk_ends):
if abs(chunk_starts[-1] - end_time) < 0.1:
# Last chunk starts at very the end? nah.
del chunk_starts[-1]
else:
# Finished with non-silence.
chunk_ends.append(end_time)
return list(zip(chunk_starts, chunk_ends))
def parse_demo(path):
jmodified = False
@ -243,13 +187,6 @@ def parse_demo(path):
db.session.rollback()
print('deadlock 5')
# calculate player voice chunks/timing if possible and not exists
if obj['voicetime'] > 0 and 'voice_chunks' not in obj:
voicefile = DEMO_PATH_PREFIX + path + '/voice/' + guid + '.opus'
if os.path.isfile(voicefile):
obj['voice_chunks'] = get_chunk_times(voicefile)
jmodified = True
# insert new player session
player_session = models.PlayerSession(
player_guid=guid,
@ -258,8 +195,7 @@ def parse_demo(path):
chats=obj['chats'],
deaths=obj['deaths'],
kills=obj['kills'],
voicetime=obj['voicetime'],
voice_chunks=obj['voice_chunks'] if 'voice_chunks' in obj else None
voicetime=obj['voicetime']
)
db.session.add(player_session)
db.session.commit()
@ -285,12 +221,18 @@ def parse_demo(path):
elif obj['winner'] == 3:
session.ct_wins += 1
# remove redundant info
eventdata = obj.copy()
del eventdata['tick']
del eventdata['event']
event = models.Event(
player_guid=guid,
session_id=session.id,
time=starttime + timedelta(seconds=obj['tick'] / tickinterval),
tick=obj['tick'],
time=starttime + timedelta(seconds=obj['tick'] * tickinterval),
event=obj['event'],
data=obj
data=eventdata
)
db.session.add(event)
db.session.commit()
@ -318,7 +260,8 @@ def parse_demo(path):
chat = models.Chat(
player_guid=obj['steamid'],
session_id=session.id,
time=starttime + timedelta(seconds=obj['tick'] / tickinterval),
tick=obj['tick'],
time=starttime + timedelta(seconds=obj['tick'] * tickinterval),
name=nick,
chat=msg
)
@ -361,7 +304,7 @@ if __name__ == '__main__':
with open('done.txt', 'r') as fp:
done = set(fp.read().splitlines())
if True:
if False:
done = set()
app = create_app()
with app.app_context():

View File

@ -1,4 +1,5 @@
click==8.0.0
dictalchemy==0.1.2.7
Flask==2.0.0
Flask-SQLAlchemy==2.5.1
gevent==21.1.2