progress

app.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
-import gevent.monkey  # noqa isort:skip
+#import gevent.monkey  # noqa isort:skip
-gevent.monkey.patch_all()  # noqa isort:skip
+#gevent.monkey.patch_all()  # noqa isort:skip
 
 from demweb.app import create_app
 

demopus.py

@@ -0,0 +1,34 @@
+import struct
+import sys
+
+with open(sys.argv[1], 'rb') as fp:
+	data = fp.read()
+
+ofs = 16
+counter = 0
+
+while ofs < len(data):
+	header = struct.unpack('B', data[ofs:ofs+1])[0]
+	ofs += 1
+	print(f'#{ofs} Header: {header}')
+
+	if header == 0x01:
+		srate = struct.unpack('I', data[ofs:ofs+4])[0]
+		ofs += 4
+		samples = struct.unpack('Q', data[ofs:ofs+8])[0]
+		ofs += 8
+		print(f'srate: {srate} | samples: {samples}\n')
+	elif header == 0x02:
+		dlen = struct.unpack('Q', data[ofs:ofs+8])[0]
+		ofs += 8
+		with open(f'{counter}.opus', 'wb') as fp:
+			fp.write(data[ofs:ofs+dlen])
+		counter += 1
+		ofs += dlen
+		print(f'opus len {dlen}')
+	elif header == 0x03:
+		silence = struct.unpack('Q', data[ofs:ofs+8])[0]
+		ofs += 8
+		print(f'silence {silence}')
+	elif header == 0x04:
+		break

demweb/app.py

@@ -10,10 +10,19 @@ def register_extensions(app):
     db.init_app(app)
 
 
+def register_jinja2_filters(app):
+    import orjson
+    with app.app_context():
+        @app.template_filter('to_json')
+        def to_json(value):
+            return orjson.dumps(value).decode('utf-8')
+
+
 def register_blueprints(app):
     with app.app_context():
         # Register blueprints
-        pass
+        from .views import bp as views_bp
+        app.register_blueprint(views_bp, url_prefix='/')
 
 
 def setup_logging(app):
@@ -77,6 +86,7 @@ def create_app(test_config=None):
         app.config.update(test_config)
 
     register_extensions(app)
+    register_jinja2_filters(app)
     register_blueprints(app)
     register_shell_context(app)
     setup_logging(app)

demweb/extensions.py

@@ -1,8 +1,10 @@
 from flask_sqlalchemy import SQLAlchemy
+from dictalchemy import make_class_dictable
 
 import config
 
 db = SQLAlchemy()
+make_class_dictable(db.Model)
 
 default_config = dict()
 for key in dir(config):

demweb/models.py

@@ -62,13 +62,13 @@ class PlayerSession(db.Model):
     deaths = db.Column(db.Integer)
     kills = db.Column(db.Integer)
     voicetime = db.Column(db.Float)
-    voice_chunks = db.Column(db.JSON(65535))
 
 
 class Chat(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     player_guid = db.Column(db.String(32), db.ForeignKey('player.guid'))
     session_id = db.Column(db.Integer, db.ForeignKey('session.id'))
+    tick = db.Column(db.Integer)
     time = db.Column(db.DateTime, index=True)
     name = db.Column(db.String(255))
     chat = db.Column(db.String(1024))
@@ -78,6 +78,7 @@ class Event(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     player_guid = db.Column(db.String(32), db.ForeignKey('player.guid'))
     session_id = db.Column(db.Integer, db.ForeignKey('session.id'))
+    tick = db.Column(db.Integer)
     time = db.Column(db.DateTime, index=True)
     event = db.Column(db.String(32), index=True)
     data = db.Column(db.JSON(1024))

demweb/static/css-ze-parsed

@@ -0,0 +1 @@
+/home/david/Projects/demboyz/premake/gmake

demweb/static/css/main.css

@@ -0,0 +1,40 @@
+
+.btn-group-xs > .btn, .btn-xs { padding: .25rem .4rem; font-size: .875rem; line-height: .5; border-radius: .2rem; }
+
+.playlist { margin: 2em 0; }
+
+.playlist .playlist-time-scale { height: 30px; }
+
+.playlist .playlist-tracks { background: #e0eff1; }
+
+.playlist .channel { background: grey; }
+
+.playlist .channel-progress { background: orange; }
+
+.playlist .cursor { background: black; }
+
+.playlist .wp-fade { background-color: rgba(0, 0, 0, 0.1); }
+
+.playlist .state-cursor, .playlist .state-select { cursor: text; }
+
+.playlist .state-fadein { cursor: w-resize; }
+
+.playlist .state-fadeout { cursor: e-resize; }
+
+.playlist .state-shift { cursor: ew-resize; }
+
+.playlist .selection.point { background: red; }
+
+.playlist .selection.segment { background: rgba(0, 0, 0, 0.1); }
+
+.playlist .channel-wrapper.silent .channel { opacity: 0.3; }
+
+.playlist .controls { background: white; text-align: center; border: 1px solid black; border-radius: 0.2rem; }
+
+.playlist .controls .track-header { overflow: hidden; color: black; height: 26px; display: flex; align-items: center; justify-content: space-between; padding: 0 0.2rem; font-size: 0.65rem; margin-bottom: -10px; }
+
+.playlist .controls .track-header button { width: 20px; height: 20px; display: flex; align-items: center; justify-content: center; }
+
+.playlist .controls input[type="range"] { display: inline-block; width: 90%; }
+
+.playlist .controls .user-info { word-break: break-word; color: black; align-items: center; justify-content: space-between; padding: 0 0.2rem; font-size: 0.65rem; margin-top: -5px; }

demweb/static/js/main.js

@@ -0,0 +1,129 @@
+var playlist = WaveformPlaylist.init({
+  container: document.getElementById("playlist"),
+  timescale: true,
+  state: 'cursor',
+  samplesPerPixel: 16384,
+  zoomLevels: [2048, 4096, 8192, 16384],
+
+  controls: {
+    show: true,
+    width: 150,
+    widgets: {
+      stereoPan: false,
+    }
+  },
+  waveHeight: 96,
+});
+
+
+function updateTrackInfo(guid, info) {
+  var tracks = $(".playlist-tracks").children();
+  for (var i = 0; i < tracks.length; i++) {
+    var track = tracks[i].firstChild;
+    var trackGuid = track.firstChild.getElementsByTagName('span')[0].innerText;
+    var userInfoElem = track.lastChild;
+    if (userInfoElem.className != 'user-info') {
+      userInfoElem = document.createElement('label');
+      userInfoElem.className = 'user-info';
+      track.appendChild(userInfoElem);
+    }
+    if (guid === null || trackGuid == guid) {
+      userInfoElem.innerText = info;
+    }
+  }
+}
+
+var ee = playlist.getEventEmitter();
+var $container = $("body");
+var $time = $container.find(".audio-pos");
+var $time_ = $container.find(".audio-pos-2");
+
+var audioPos = 0;
+
+function clockFormat(seconds, decimals) {
+  var hours,
+      minutes,
+      secs,
+      result;
+
+  hours = parseInt(seconds / 3600, 10) % 24;
+  minutes = parseInt(seconds / 60, 10) % 60;
+  secs = seconds % 60;
+  secs = secs.toFixed(decimals);
+
+  result = (hours < 10 ? "0" + hours : hours) + ":" + (minutes < 10 ? "0" + minutes : minutes) + ":" + (secs < 10 ? "0" + secs : secs);
+
+  return result;
+}
+
+function updateTime(time) {
+  $time.html(clockFormat(time, 3));
+  audioPos = time;
+
+  var tick = time / g_session.tickinterval;
+  var silenceTicks = 0;
+  for (var i = 0; i < g_session.silence_chunks.length; i++) {
+    var chunk = g_session.silence_chunks[i];
+    if (tick > chunk[0]) {
+      silenceTicks += chunk[1];
+    } else {
+      break;
+    }
+  }
+  var tickedTime = (tick + silenceTicks) * g_session.tickinterval;
+  $time_.html(clockFormat(tickedTime, 3));
+}
+
+updateTime(audioPos);
+
+$container.on("click", ".btn-play", function() {
+  ee.emit("play");
+});
+
+$container.on("click", ".btn-pause", function() {
+  isLooping = false;
+  ee.emit("pause");
+});
+
+$container.on("click", ".btn-stop", function() {
+  isLooping = false;
+  ee.emit("stop");
+});
+
+$container.on("click", ".btn-rewind", function() {
+  isLooping = false;
+  ee.emit("rewind");
+});
+
+$container.on("click", ".btn-fast-forward", function() {
+  isLooping = false;
+  ee.emit("fastforward");
+});
+
+//zoom buttons
+$container.on("click", ".btn-zoom-in", function() {
+  ee.emit("zoomin");
+});
+
+$container.on("click", ".btn-zoom-out", function() {
+  ee.emit("zoomout");
+});
+
+
+$container.on("input change", ".master-gain", function(e){
+  ee.emit("mastervolumechange", e.target.value);
+});
+$container.find(".master-gain").change();
+
+$container.on("change", ".automatic-scroll", function(e){
+  ee.emit("automaticscroll", $(e.target).is(':checked'));
+});
+$container.find(".automatic-scroll").change();
+
+
+ee.on("timeupdate", updateTime);
+
+
+function onFinishedLoading() {
+  updateTrackInfo(null, "name");
+}

demweb/templates/session.html

@@ -0,0 +1,166 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+  <meta charset="utf-8" />
+  <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+  <meta name="viewport" content="width=device-width, initial-scale=1" />
+
+  <title>
+    {{ session.demoname }}
+  </title>
+
+  <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.1/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-+0n0xVW2eSR5OomGNYDnhzAbDsOXxcvSN1TPprVMTNDbiYZCxYbOOl7+AMvyTG2x" crossorigin="anonymous">
+  <link rel="stylesheet" href="/static/css/main.css">
+  <script src="https://kit.fontawesome.com/ef69927139.js" crossorigin="anonymous"></script>
+</head>
+
+
+<body>
+<nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4">
+  <div class="container-fluid">
+    <a class="navbar-brand" href="#">Top navbar</a>
+    <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" aria-expanded="false" aria-label="Toggle navigation">
+      <span class="navbar-toggler-icon"></span>
+    </button>
+    <div class="collapse navbar-collapse" id="navbarCollapse">
+      <ul class="navbar-nav me-auto mb-2 mb-md-0">
+        <li class="nav-item">
+          <a class="nav-link active" aria-current="page" href="#">Home</a>
+        </li>
+        <li class="nav-item">
+          <a class="nav-link" href="#">Link</a>
+        </li>
+        <li class="nav-item">
+          <a class="nav-link disabled" href="#" tabindex="-1" aria-disabled="true">Disabled</a>
+        </li>
+      </ul>
+      <form class="d-flex">
+        <input class="form-control me-2" type="search" placeholder="Search" aria-label="Search">
+        <button class="btn btn-outline-success" type="submit">Search</button>
+      </form>
+    </div>
+  </div>
+</nav>
+
+<main class="container-fluid">
+
+<div class="btn-toolbar" role="toolbar">
+  <div class="btn-group me-2" role="group">
+    <button type="button" class="btn-pause btn btn-outline-warning" title="Pause">
+      <i class="fas fa-pause"></i>
+    </button>
+    <button type="button" class="btn-play btn btn-outline-success" title="Play">
+      <i class="fas fa-play"></i>
+    </button>
+    <button type="button" class="btn-stop btn btn-outline-danger" title="Stop">
+      <i class="fas fa-stop"></i>
+    </button>
+    <button
+      type="button"
+      class="btn-rewind btn btn-outline-success"
+      title="Rewind"
+    >
+      <i class="fas fa-fast-backward"></i>
+    </button>
+    <button
+      type="button"
+      class="btn-fast-forward btn btn-outline-success"
+      title="Fast forward"
+    >
+      <i class="fas fa-fast-forward"></i>
+    </button>
+  </div>
+
+  <div class="btn-group me-2" role="group">
+    <button type="button" title="Zoom in" class="btn-zoom-in btn btn-outline-dark">
+      <i class="fas fa-search-plus" aria-hidden="true"></i>
+    </button>
+    <button type="button" title="Zoom out" class="btn-zoom-out btn btn-outline-dark">
+      <i class="fas fa-search-minus" aria-hidden="true"></i>
+    </button>
+  </div>
+
+  <div class="btn-group me-2">
+    <div style="margin: 6px">
+      <input
+        type="range"
+        min="0"
+        max="100"
+        value="50"
+        class="master-gain form-range mw-50"
+        id="master-gain"
+      />
+    </div>
+
+    <div style="margin: 6px">
+      <span class="audio-pos" aria-label="Audio position">00:00:00.0</span>
+    </div>
+
+    <div style="margin: 6px">
+      <span class="audio-pos-2" aria-label="Audio position">00:00:00.0</span>
+    </div>
+
+    <div class="form-check form-switch">
+      <input class="form-check-input automatic-scroll" type="checkbox" id="automatic_scroll" checked>
+      <label class="form-check-label" for="automatic_scroll">Autoscroll</label>
+    </div>
+  </div>
+
+</div>
+
+<div id="playlist">
+</div>
+
+</main>
+
+
+<script
+  src="https://code.jquery.com/jquery-3.6.0.min.js"
+  integrity="sha256-/xUj+3OJU5yExlq6GSYGSHk7tPXikynS7ogEvDej/m4="
+  crossorigin="anonymous"
+></script>
+
+<script
+  src="/static/js/waveform-playlist.var.js?v=4.0.1"
+></script>
+
+<script type="text/javascript">
+var g_session = {{ session.asdict() | to_json | safe }};
+
+var g_player_sessions = {
+{%- for guid, psess in player_sessions.items() %}
+"{{ guid }}": {{ psess.asdict(exclude=['player_guid', 'session_id']) | to_json | safe }},
+{%- endfor %}
+};
+
+var g_chats = [
+{%- for chat in chats %}
+{{ chat.asdict(exclude=['id', 'session_id']) | to_json | safe }},
+{%- endfor %}
+];
+
+var g_events = [
+{%- for event in events %}
+{{ event.asdict(exclude=['id', 'session_id']) | to_json | safe }},
+{%- endfor %}
+];
+</script>
+
+<script
+  src="/static/js/main.js"
+></script>
+
+<script type="text/javascript">
+playlist.load([
+{%- for guid, psess in player_sessions.items() %}
+{%- if psess.voicetime > 0 %}
+{src: "/static/css-ze-parsed/{{ session.demoname }}/voice/{{ guid }}.demopus", name: "{{ guid }}"},
+{%- endif -%}
+{% endfor %}
+]).then(function() {
+  onFinishedLoading();
+});
+</script>
+
+</body>
+</html>

demweb/views.py

@@ -0,0 +1,42 @@
+from flask import Blueprint, render_template
+
+from demweb.extensions import db
+from demweb.models import *
+
+bp = Blueprint('views', __name__)
+
+@bp.route('')
+def home():
+    return 'hello world'
+
+@bp.route('/session/<int:session_id>')
+def session(session_id):
+    session = Session.query.filter_by(id=session_id).one()
+    if not session:
+        return '404 Not Found', 404
+
+    player_sessions_ = PlayerSession.query.filter_by(session_id=session_id).all()
+    player_sessions = dict()
+    for psess in player_sessions_:
+        player_sessions[psess.player_guid] = psess
+    del player_sessions_
+
+    players_ = Player.query.filter(Player.guid.in_(player_sessions.keys()))
+    players = dict()
+    for player in players_:
+        players[player.guid] = player
+    del players_
+
+    chats = Chat.query.filter_by(session_id=session_id).all()
+    chats.sort(key=lambda x: x.tick)
+
+    events = Event.query.filter_by(session_id=session_id).all()
+    events.sort(key=lambda x: x.tick)
+
+    return render_template('session.html',
+        session=session,
+        player_sessions=player_sessions,
+        players=players,
+        chats=chats,
+        events=events
+    )

parser.py

@@ -7,7 +7,6 @@ import multiprocessing
 import signal
 import re
 import traceback
-import subprocess
 from datetime import datetime, timedelta
 from sqlalchemy import exc, func
 from sqlalchemy.sql.expression import bindparam
@@ -45,61 +44,6 @@ def remove_chatcolors(message):
         i += 1
     return output
 
-silence_start_re = re.compile(' silence_start: (?P<start>[0-9]+(\.?[0-9]*))$')
-silence_end_re = re.compile(' silence_end: (?P<end>[0-9]+(\.?[0-9]*)) ')
-total_duration_re = re.compile('size=[^ ]+ time=(?P<hours>[0-9]{2}):(?P<minutes>[0-9]{2}):(?P<seconds>[0-9\.]{5}) bitrate=')
-
-def get_chunk_times(path):
-    proc = subprocess.Popen([
-        'ffmpeg',
-        '-i', path,
-        '-af', 'silencedetect=n=-80dB:d=0.1',
-        '-f', 'null',
-        '-'],
-        stderr=subprocess.PIPE,
-        stdout=subprocess.DEVNULL
-    )
-    output = proc.communicate()[1].decode('utf-8')
-    lines = output.splitlines()
-
-    # Chunks start when silence ends, and chunks end when silence starts.
-    chunk_starts = []
-    chunk_ends = []
-    for line in lines:
-        silence_start_match = silence_start_re.search(line)
-        silence_end_match = silence_end_re.search(line)
-        total_duration_match = total_duration_re.search(line)
-        if silence_start_match:
-            start = float(silence_start_match.group('start'))
-            if start == 0.:
-                # Ignore initial silence.
-                continue
-            chunk_ends.append(start)
-            if len(chunk_starts) == 0:
-                # Started with non-silence.
-                chunk_starts.append(0.)
-        elif silence_end_match:
-            chunk_starts.append(float(silence_end_match.group('end')))
-        elif total_duration_match:
-            hours = int(total_duration_match.group('hours'))
-            minutes = int(total_duration_match.group('minutes'))
-            seconds = float(total_duration_match.group('seconds'))
-            end_time = hours * 3600 + minutes * 60 + seconds
-
-    if len(chunk_starts) == 0:
-        # No silence found.
-        chunk_starts.append(0)
-
-    if len(chunk_starts) > len(chunk_ends):
-        if abs(chunk_starts[-1] - end_time) < 0.1:
-            # Last chunk starts at very the end? nah.
-            del chunk_starts[-1]
-        else:
-            # Finished with non-silence.
-            chunk_ends.append(end_time)
-
-    return list(zip(chunk_starts, chunk_ends))
-
 
 def parse_demo(path):
     jmodified = False
@@ -243,13 +187,6 @@ def parse_demo(path):
                 db.session.rollback()
                 print('deadlock 5')
 
-        # calculate player voice chunks/timing if possible and not exists
-        if obj['voicetime'] > 0 and 'voice_chunks' not in obj:
-            voicefile = DEMO_PATH_PREFIX + path + '/voice/' + guid + '.opus'
-            if os.path.isfile(voicefile):
-                obj['voice_chunks'] = get_chunk_times(voicefile)
-                jmodified = True
-
         # insert new player session
         player_session = models.PlayerSession(
             player_guid=guid,
@@ -258,8 +195,7 @@ def parse_demo(path):
             chats=obj['chats'],
             deaths=obj['deaths'],
             kills=obj['kills'],
-            voicetime=obj['voicetime'],
+            voicetime=obj['voicetime']
-            voice_chunks=obj['voice_chunks'] if 'voice_chunks' in obj else None
         )
         db.session.add(player_session)
         db.session.commit()
@@ -285,12 +221,18 @@ def parse_demo(path):
             elif obj['winner'] == 3:
                 session.ct_wins += 1
 
+        # remove redundant info
+        eventdata = obj.copy()
+        del eventdata['tick']
+        del eventdata['event']
+
         event = models.Event(
             player_guid=guid,
             session_id=session.id,
-            time=starttime + timedelta(seconds=obj['tick'] / tickinterval),
+            tick=obj['tick'],
+            time=starttime + timedelta(seconds=obj['tick'] * tickinterval),
             event=obj['event'],
-            data=obj
+            data=eventdata
         )
         db.session.add(event)
     db.session.commit()
@@ -318,7 +260,8 @@ def parse_demo(path):
             chat = models.Chat(
                 player_guid=obj['steamid'],
                 session_id=session.id,
-                time=starttime + timedelta(seconds=obj['tick'] / tickinterval),
+                tick=obj['tick'],
+                time=starttime + timedelta(seconds=obj['tick'] * tickinterval),
                 name=nick,
                 chat=msg
             )
@@ -361,7 +304,7 @@ if __name__ == '__main__':
     with open('done.txt', 'r') as fp:
         done = set(fp.read().splitlines())
 
-    if True:
+    if False:
         done = set()
         app = create_app()
         with app.app_context():

requirements.txt

@@ -1,4 +1,5 @@
 click==8.0.0
+dictalchemy==0.1.2.7
 Flask==2.0.0
 Flask-SQLAlchemy==2.5.1
 gevent==21.1.2