DRUM SUPPORT!

8 files changed, 515 insertions, 43 deletions

diff --git a/broadcast.py b/broadcast.py
index 450874a..6747910 100644
--- a/broadcast.py
+++ b/broadcast.py
@@ -32,19 +32,23 @@ parser.add_option('-S', '--seek', dest='seek', type='float', help='Start time in
 parser.add_option('-f', '--factor', dest='factor', type='float', help='Rescale time by this factor (0<f<1 are faster; 0.5 is twice the speed, 2 is half)')
 parser.add_option('-r', '--route', dest='routes', action='append', help='Add a routing directive (see --route-help)')
 parser.add_option('-v', '--verbose', dest='verbose', action='store_true', help='Be verbose; dump events and actual time (can slow down performance!)')
-parser.add_option('-W', '--wait-time', dest='wait_time', type='float', help='How long to wait for clients to initially respond (delays all broadcasts)')
+parser.add_option('-W', '--wait-time', dest='wait_time', type='float', help='How long to wait between pings for clients to initially respond (delays all broadcasts)')
+parser.add_option('--tries', dest='tries', type='int', help='Number of ping packets to send')
 parser.add_option('-B', '--bind-addr', dest='bind_addr', help='The IP address (or IP:port) to bind to (influences the network to send to)')
+parser.add_option('--port', dest='ports', action='append', type='int', help='Add a port to find clients on')
+parser.add_option('--clear-ports', dest='ports', action='store_const', const=[], help='Clear ports previously specified (including the default)')
 parser.add_option('--repeat', dest='repeat', action='store_true', help='Repeat the file playlist indefinitely')
 parser.add_option('-n', '--number', dest='number', type='int', help='Number of clients to use; if negative (default -1), use the product of stream count and the absolute value of this parameter')
 parser.add_option('--dry', dest='dry', action='store_true', help='Dry run--don\'t actually search for or play to clients, but pretend they exist (useful with -G)')
 parser.add_option('--pcm', dest='pcm', action='store_true', help='Use experimental PCM rendering')
 parser.add_option('--pcm-lead', dest='pcmlead', type='float', help='Seconds of leading PCM data to send')
+parser.add_option('--spin', dest='spin', action='store_true', help='Ignore delta times in the queue (busy loop the CPU) for higher accuracy')
 parser.add_option('-G', '--gui', dest='gui', default='', help='set a GUI to use')
 parser.add_option('--pg-fullscreen', dest='fullscreen', action='store_true', help='Use a full-screen video mode')
 parser.add_option('--pg-width', dest='pg_width', type='int', help='Width of the pygame window')
 parser.add_option('--pg-height', dest='pg_height', type='int', help='Width of the pygame window')
 parser.add_option('--help-routes', dest='help_routes', action='store_true', help='Show help about routing directives')
-parser.set_defaults(routes=[], test_delay=0.25, random=0.0, rand_low=80, rand_high=2000, live=None, factor=1.0, duration=1.0, volume=1.0, wait_time=0.25, play=[], transpose=0, seek=0.0, bind_addr='', pg_width = 0, pg_height = 0, number=-1, pcmlead=0.1)
+parser.set_defaults(routes=[], test_delay=0.25, random=0.0, rand_low=80, rand_high=2000, live=None, factor=1.0, duration=1.0, volume=1.0, wait_time=0.1, tries=5, play=[], transpose=0, seek=0.0, bind_addr='', ports=[13676],  pg_width = 0, pg_height = 0, number=-1, pcmlead=0.1)
 options, args = parser.parse_args()
 
 if options.help_routes:
@@ -96,6 +100,7 @@ def gui_pygame():
     PFAC = HEIGHT / 128.0
 
     clock = pygame.time.Clock()
+    font = pygame.font.SysFont(pygame.font.get_default_font(), 24)
 
     print 'Pygame GUI initialized, running...'
 
@@ -110,6 +115,9 @@ def gui_pygame():
             col = [int(i*255) for i in col]
             disp.fill(col, (WIDTH - 1, HEIGHT - pitch * PFAC - PFAC, 1, PFAC))
             idx += 1
+        tsurf = font.render('%0.3f' % ((time.time() - BASETIME) / factor,), True, (255, 255, 255), (0, 0, 0))
+        disp.fill((0, 0, 0), tsurf.get_rect())
+        disp.blit(tsurf, (0, 0))
         pygame.display.flip()
 
         for ev in pygame.event.get():
@@ -123,7 +131,6 @@ def gui_pygame():
 
 GUIS['pygame'] = gui_pygame
 
-PORT = 13676
 factor = options.factor
 
 print 'Factor:', factor
@@ -136,26 +143,28 @@ if options.bind_addr:
         port = '12074'
     s.bind((addr, int(port)))
 
-clients = []
-targets = []
+clients = set()
+targets = set()
 uid_groups = {}
 type_groups = {}
 
 if not options.dry:
-    s.sendto(str(Packet(CMD.PING)), ('255.255.255.255', PORT))
     s.settimeout(options.wait_time)
-
-    try:
-            while True:
+    for PORT in options.ports:
+        for num in xrange(options.tries):
+            s.sendto(str(Packet(CMD.PING)), ('255.255.255.255', PORT))
+            try:
+                while True:
                     data, src = s.recvfrom(4096)
-                    clients.append(src)
-    except socket.timeout:
-            pass
+                    clients.add(src)
+            except socket.timeout:
+                pass
 
 print len(clients), 'detected clients'
 
-print 'Clients:'
-for cl in clients:
+for num in xrange(options.tries):
+    print 'Try', num
+    for cl in clients:
 	print cl,
         s.sendto(str(Packet(CMD.CAPS)), cl)
         data, _ = s.recvfrom(4096)
@@ -167,8 +176,8 @@ for cl in clients:
         print 'uid', uid
         if uid == '':
             uid = None
-        uid_groups.setdefault(uid, []).append(cl)
-        type_groups.setdefault(tp, []).append(cl)
+        uid_groups.setdefault(uid, set()).add(cl)
+        type_groups.setdefault(tp, set()).add(cl)
 	if options.test:
                 ts, tms = int(options.test_delay), int(options.test_delay * 1000000) % 1000000
 		s.sendto(str(Packet(CMD.PLAY, ts, tms, 440, options.volume)), cl)
@@ -180,7 +189,7 @@ for cl in clients:
         if options.silence:
                 s.sendto(str(Packet(CMD.PLAY, 0, 1, 1, 0.0)), cl)
         for i in xrange(pkt.data[0]):
-            targets.append(cl+(i,))
+            targets.add(cl+(i,))
 
 playing_notes = {}
 for tg in targets:
@@ -418,11 +427,11 @@ for fname in args:
     class RouteSet(object):
         def __init__(self, clis=None):
             if clis is None:
-                clis = targets[:]
+                clis = set(targets)
             self.clients = clis
             self.routes = []
         def Route(self, stream):
-            testset = self.clients[:]
+            testset = set(self.clients)
             grp = stream.get('group', 'ALL')
             if options.verbose:
                 print 'Routing', grp, '...'
@@ -459,7 +468,7 @@ for fname in args:
                 if options.verbose:
                     print '\tOut of clients, no route matched.'
                 return None
-            cli = testset[0]
+            cli = list(testset)[0]
             self.clients.remove(cli)
             if options.verbose:
                 print '\tDefault route to', cli
@@ -479,6 +488,50 @@ for fname in args:
             print route
 
     class NSThread(threading.Thread):
+            def __init__(self, *args, **kwargs):
+                threading.Thread.__init__(self, *args, **kwargs)
+                self.done = False
+                self.cur_offt = None
+                self.next_t = None
+            def actuate_missed(self):
+                nsq, cls = self._Thread__args
+                dur = None
+                i = 0
+                while nsq and float(nsq[0].get('time'))*factor <= time.time() - BASETIME:
+                    i += 1
+                    note = nsq.pop(0)
+                    ttime = float(note.get('time'))
+                    pitch = float(note.get('pitch')) + options.transpose
+                    ampl = float(note.get('ampl', float(note.get('vel', 127.0)) / 127.0))
+                    dur = factor*float(note.get('dur'))
+                    if options.verbose:
+                        print (time.time() - BASETIME) / options.factor, ': PLAY', pitch, dur, ampl
+                    if options.dry:
+                        playing_notes[self.ident] = (pitch, ampl)
+                    else:
+                        for cl in cls:
+                            s.sendto(str(Packet(CMD.PLAY, int(dur), int((dur*1000000)%1000000), int(440.0 * 2**((pitch-69)/12.0)), ampl * options.volume, cl[2])), cl[:2])
+                            playing_notes[cl] = (pitch, ampl)
+                if i > 0 and dur is not None:
+                    self.cur_offt = ttime + dur
+                else:
+                    if self.cur_offt:
+                        if factor * self.cur_offt <= time.time() - BASETIME:
+                            if options.verbose:
+                                print '% 6.5f'%((time.time() - BASETIME) / factor,), ': DONE'
+                            self.cur_offt = None
+                            if options.dry:
+                                playing_notes[self.ident] = (0, 0)
+                            else:
+                                for cl in cls:
+                                    playing_notes[cl] = (0, 0)
+                next_act = None
+                if nsq:
+                    next_act = float(nsq[0].get('time'))
+                if options.verbose:
+                    print 'NEXT_ACT:', next_act, 'CUR_OFFT:', self.cur_offt
+                self.next_t = min((next_act or float('inf'), self.cur_offt or float('inf')))
+                self.done = not (nsq or self.cur_offt)
             def drop_missed(self):
                 nsq, cl = self._Thread__args
                 cnt = 0
@@ -487,7 +540,6 @@ for fname in args:
                     cnt += 1
                 if options.verbose:
                     print self, 'dropped', cnt, 'notes due to miss'
-                self._Thread__args = (nsq, cl)
             def wait_for(self, t):
                 if t <= 0:
                     return
@@ -518,6 +570,7 @@ for fname in args:
     if options.dry:
         for ns in notestreams:
             nsq = ns.findall('note')
+            nsq.sort(key=lambda x: float(x.get('time')))
             threads[ns] = NSThread(args=(nsq, set()))
         targets = threads.values()  # XXX hack
     else:
@@ -526,6 +579,7 @@ for fname in args:
             cli = routeset.Route(ns)
             if cli:
                 nsq = ns.findall('note')
+                nsq.sort(key=lambda x: float(x.get('time')))
                 if ns in threads:
                     threads[ns]._Thread__args[1].add(cli)
                 else:
@@ -540,8 +594,16 @@ for fname in args:
     if options.seek > 0:
         for thr in threads.values():
             thr.drop_missed()
-    for thr in threads.values():
-            thr.start()
-    for thr in threads.values():
-            thr.join()
+    while not all(thr.done for thr in threads.values()):
+        for thr in threads.values():
+            if thr.next_t is None or factor * thr.next_t <= time.time() - BASETIME:
+                thr.actuate_missed()
+        delta = factor * min(thr.next_t for thr in threads.values() if thr.next_t is not None) + BASETIME - time.time()
+        if delta == float('inf'):
+            print 'WARNING: Infinite postponement detected! Did all notestreams finish?'
+            break
+        if options.verbose:
+            print 'TICK DELTA:', delta
+        if delta >= 0 and not options.spin:
+            time.sleep(delta)
     print fname, ': Done!'
diff --git a/client.py b/client.py
index 8bd41a4..855fb4b 100644
--- a/client.py
+++ b/client.py
@@ -294,7 +294,7 @@ if options.numpy:
     def samps(freq, amp, phase, cnt):
         samps = numpy.ndarray((cnt,), numpy.int32)
         pvel = 2 * math.pi * freq / RATE
-        fac = amp / float(STREAMS)
+        fac = options.volume * amp / float(STREAMS)
         for i in xrange(cnt):
             samps[i] = fac * max(-1, min(1, generator(phase)))
             phase = (phase + pvel) % (2 * math.pi)
@@ -414,7 +414,7 @@ while True:
         data = [0] * 8
         data[0] = STREAMS
         data[1] = stoi(IDENT)
-        for i in xrange(len(UID)/4):
+        for i in xrange(len(UID)/4 + 1):
             data[i+2] = stoi(UID[4*i:4*(i+1)])
         sock.sendto(str(Packet(CMD.CAPS, *data)), cli)
     elif pkt.cmd == CMD.PCM:
diff --git a/drums.py b/drums.py
new file mode 100644
index 0000000..3f2ab9f
--- /dev/null
+++ b/drums.py
@@ -0,0 +1,180 @@
+import pyaudio
+import socket
+import optparse
+import tarfile
+import wave
+import cStringIO as StringIO
+import array
+import time
+
+from packet import Packet, CMD, stoi
+
+parser = optparse.OptionParser()
+parser.add_option('-t', '--test', dest='test', action='store_true', help='As a test, play all samples then exit')
+parser.add_option('-v', '--verbose', dest='verbose', action='store_true', help='Be verbose')
+parser.add_option('-V', '--volume', dest='volume', type='float', default=1.0, help='Set the volume factor (nominally [0.0, 1.0], but >1.0 can be used to amplify with possible distortion)')
+parser.add_option('-r', '--rate', dest='rate', type='int', default=44100, help='Audio sample rate for output and of input files')
+parser.add_option('-u', '--uid', dest='uid', default='', help='User identifier of this client')
+parser.add_option('-p', '--port', dest='port', default=13676, type='int', help='UDP port to listen on')
+parser.add_option('--repeat', dest='repeat', action='store_true', help='If a note plays longer than a sample length, keep playing the sample')
+parser.add_option('--cut', dest='cut', action='store_true', help='If a note ends within a sample, stop playing that sample immediately')
+
+options, args = parser.parse_args()
+
+MAX = 0x7fffffff
+MIN = -0x80000000
+IDENT = 'DRUM'
+
+if not args:
+    print 'Need at least one drumpack (.tar.bz2) as an argument!'
+    parser.print_usage()
+    exit(1)
+
+DRUMS = {}
+
+for fname in args:
+    print 'Reading', fname, '...'
+    tf = tarfile.open(fname, 'r')
+    names = tf.getnames()
+    for nm in names:
+        if not (nm.endswith('.wav') or nm.endswith('.raw')) or len(nm) < 5:
+            continue
+        frq = int(nm[:-4])
+        if options.verbose:
+            print '\tLoading frq', frq, '...'
+        fo = tf.extractfile(nm)
+        if nm.endswith('.wav'):
+            wf = wave.open(fo)
+            if wf.getnchannels() != 1:
+                print '\t\tWARNING: Channel count wrong: got', wf.getnchannels(), 'expecting 1'
+            if wf.getsampwidth() != 4:
+                print '\t\tWARNING: Sample width wrong: got', wf.getsampwidth(), 'expecting 4'
+            if wf.getframerate() != options.rate:
+                print '\t\tWARNING: Rate wrong: got', wf.getframerate(), 'expecting', options.rate, '(maybe try setting -r?)'
+            frames = wf.getnframes()
+            data = ''
+            while len(data) < wf.getsampwidth() * frames:
+                data += wf.readframes(frames - len(data) / wf.getsampwidth())
+        elif nm.endswith('.raw'):
+            data = fo.read()
+            frames = len(data) / 4
+        if options.verbose:
+            print '\t\tData:', frames, 'samples,', len(data), 'bytes'
+        if frq in DRUMS:
+            print '\t\tWARNING: frequency', frq, 'already in map, overwriting...'
+        DRUMS[frq] = data
+
+if options.verbose:
+    print len(DRUMS), 'sounds loaded'
+
+PLAYING = set()
+
+class SampleReader(object):
+    def __init__(self, buf, total, amp):
+        self.buf = buf
+        self.total = total
+        self.cur = 0
+        self.amp = amp
+
+    def read(self, bytes):
+        if self.cur >= self.total:
+            return ''
+        res = ''
+        while self.cur < self.total and len(res) < bytes:
+            data = self.buf[self.cur % len(self.buf):self.cur % len(self.buf) + bytes - len(res)]
+            self.cur += len(data)
+            res += data
+        arr = array.array('i')
+        arr.fromstring(res)
+        for i in range(len(arr)):
+            arr[i] = int(arr[i] * self.amp)
+        return arr.tostring()
+
+    def __repr__(self):
+        return '<SR (%d) @%d / %d A:%f>'%(len(self.buf), self.cur, self.total, self.amp)
+
+def gen_data(data, frames, tm, status):
+    fdata = array.array('l', [0] * frames)
+    torem = set()
+    for src in set(PLAYING):
+        buf = src.read(frames * 4)
+        if not buf:
+            torem.add(src)
+            continue
+        samps = array.array('i')
+        samps.fromstring(buf)
+        if len(samps) < frames:
+            samps.extend([0] * (frames - len(samps)))
+        for i in range(frames):
+            fdata[i] += samps[i]
+    for src in torem:
+        PLAYING.discard(src)
+    for i in range(frames):
+        fdata[i] = max(MIN, min(MAX, fdata[i]))
+    fdata = array.array('i', fdata)
+    return (fdata.tostring(), pyaudio.paContinue)
+
+pa = pyaudio.PyAudio()
+stream = pa.open(rate=options.rate, channels=1, format=pyaudio.paInt32, output=True, frames_per_buffer=64, stream_callback=gen_data)
+
+if options.test:
+    for frq in sorted(DRUMS.keys()):
+        print 'Current playing:', PLAYING
+        print 'Playing:', frq
+        data = DRUMS[frq]
+        PLAYING.add(SampleReader(data, len(data), 1.0))
+        time.sleep(len(data) / (4.0 * options.rate))
+    print 'Done'
+    exit()
+
+
+sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+sock.bind(('', options.port))
+
+#signal.signal(signal.SIGALRM, sigalrm)
+
+while True:
+    data = ''
+    while not data:
+        try:
+            data, cli = sock.recvfrom(4096)
+        except socket.error:
+            pass
+    pkt = Packet.FromStr(data)
+    print 'From', cli, 'command', pkt.cmd
+    if pkt.cmd == CMD.KA:
+        pass
+    elif pkt.cmd == CMD.PING:
+        sock.sendto(data, cli)
+    elif pkt.cmd == CMD.QUIT:
+        break
+    elif pkt.cmd == CMD.PLAY:
+        frq = pkt.data[2]
+        if frq not in DRUMS:
+            print 'WARNING: No such instrument', frq, ', ignoring...'
+            continue
+        rdata = DRUMS[frq]
+        rframes = len(rdata) / 4
+        dur = pkt.data[0]+pkt.data[1]/1000000.0
+        dframes = int(dur * options.rate)
+        if not options.repeat:
+            dframes = max(dframes, rframes)
+        if not options.cut:
+            dframes = rframes * ((dframes + rframes - 1) / rframes)
+        amp = max(min(pkt.as_float(3), 1.0), 0.0)
+        PLAYING.add(SampleReader(rdata, dframes * 4, amp))
+        #signal.setitimer(signal.ITIMER_REAL, dur)
+    elif pkt.cmd == CMD.CAPS:
+        data = [0] * 8
+        data[0] = 255  # XXX More ports? Less?
+        data[1] = stoi(IDENT)
+        for i in xrange(len(options.uid)/4 + 1):
+            data[i+2] = stoi(options.uid[4*i:4*(i+1)])
+        sock.sendto(str(Packet(CMD.CAPS, *data)), cli)
+#    elif pkt.cmd == CMD.PCM:
+#        fdata = data[4:]
+#        fdata = struct.pack('16i', *[i<<16 for i in struct.unpack('16h', fdata)])
+#        QUEUED_PCM += fdata
+#        print 'Now', len(QUEUED_PCM) / 4.0, 'frames queued'
+    else:
+        print 'Unknown cmd', pkt.cmd
diff --git a/drums.tar.bz2 b/drums.tar.bz2
new file mode 100644
index 0000000..d769e1b
--- /dev/null
+++ b/drums.tar.bz2
diff --git a/make_patfile.sh b/make_patfile.sh
new file mode 100755
index 0000000..b656d2d
--- /dev/null
+++ b/make_patfile.sh
@@ -0,0 +1,29 @@
+# Convert the FreePats Drums_000 directory to ITL Chorus drums.tar.bz2
+# Note: technically, this must be run in a directory with subdirectories
+# starting with a MIDI pitch, and containing a text file with a
+# "convert_to_wav:" command that produces a .wav in that working directory.
+# sox is required to convert the audio. This handles the dirt-old options
+# for sox in the text files explicitly to support the FreePats standard.
+# The current version was checked in from the Drums_000 directory to be
+# found in the TAR at this URL:
+# http://freepats.zenvoid.org/samples/freepats/freepats-raw-samples.tar.bz2
+# Thank you again, FreePats!
+
+rm *.wav .wav *.raw .raw
+
+for i in *; do
+	if [ -d $i ]; then
+		pushd $i
+		eval `grep 'convert_to_wav' *.txt | sed -e 's/convert_to_wav: //' -e 's/-w/-b 16/' -e 's/-s/-e signed/' -e 's/-u/-e unsigned/'`
+		PITCH=`echo "$i" | sed -e 's/^\([0-9]\+\).*$/\1/g'`
+		# From broadcast.py, eval'd in Python for consistent results
+		FRQ=`echo $PITCH | python2 -c "print(int(440.0 * 2**((int(raw_input())-69)/12.0)))"`
+		echo "WRITING $FRQ.wav"
+		[ -z "$FRQ" ] && echo "!!! EMPTY FILENAME?"
+		sox *.wav -r 44100 -c 1 -e signed -b 32 -t raw ../$FRQ.raw
+		popd
+	fi
+done
+
+rm drums.tar.bz2
+tar cjf drums.tar.bz2 *.raw
diff --git a/mkiv.py b/mkiv.py
index 867a933..3eca1e9 100644
--- a/mkiv.py
+++ b/mkiv.py
@@ -39,9 +39,16 @@ parser.add_option('--modwheel-amp-dev', dest='modadev', type='float', help='Devi
 parser.add_option('--modwheel-amp-freq', dest='modafreq', type='float', help='Frequency of modulation periods (sinusoids) of the modwheel acting on amplitude')
 parser.add_option('--modwheel-res', dest='modres', type='float', help='(Fractional) seconds by which to resolve modwheel events (0 to disable)')
 parser.add_option('--modwheel-continuous', dest='modcont', action='store_true', help='Keep phase continuous in global time (don\'t reset to 0 for each note)')
+parser.add_option('--string-res', dest='stringres', type='float', help='(Fractional) seconds by which to resolve string models (0 to disable)')
+parser.add_option('--string-max', dest='stringmax', type='int', help='Maximum number of events to generate per single input event')
+parser.add_option('--string-rate-on', dest='stringonrate', type='float', help='Rate (amplitude / sec) by which to exponentially decay in the string model while a note is active')
+parser.add_option('--string-rate-off', dest='stringoffrate', type='float', help='Rate (amplitude / sec) by which to exponentially decay in the string model after a note ends')
+parser.add_option('--string-threshold', dest='stringthres', type='float', help='Amplitude (as fraction of original) at which point the string model event is terminated')
 parser.add_option('--tempo', dest='tempo', help='Adjust interpretation of tempo (try "f1"/"global", "f2"/"track")')
+parser.add_option('--epsilon', dest='epsilon', type='float', help='Don\'t consider overlaps smaller than this number of seconds (which regularly happen due to precision loss)')
+parser.add_option('--vol-pow', dest='vol_pow', type='float', help='Exponent to raise volume changes (adjusts energy per delta volume)')
 parser.add_option('-0', '--keep-empty', dest='keepempty', action='store_true', help='Keep (do not cull) events with 0 duration in the output file')
-parser.set_defaults(tracks=[], perc='GM', deviation=2, tempo='global', modres=0.005, modfdev=2.0, modffreq=8.0, modadev=0.5, modafreq=8.0)
+parser.set_defaults(tracks=[], perc='GM', deviation=2, tempo='global', modres=0.005, modfdev=2.0, modffreq=8.0, modadev=0.5, modafreq=8.0, stringres=0, stringmax=1024, stringrateon=0.7, stringrateoff=0.01, stringthres=0.02, epsilon=1e-12, vol_pow=2)
 options, args = parser.parse_args()
 if options.tempo == 'f1':
     options.tempo == 'global'
@@ -238,6 +245,8 @@ for fname in args:
         def __repr__(self):
             return '<ME %r in %d on (%d:%d) MW:%d @%f>'%(self.ev, self.tidx, self.bank, self.prog, self.mw, self.abstime)
 
+    vol_at = [[{0: 0x3FFF} for i in range(16)] for j in range(len(pat))]
+
     events = []
     cur_mw = [[0 for i in range(16)] for j in range(len(pat))]
     cur_bank = [[0 for i in range(16)] for j in range(len(pat))]
@@ -245,6 +254,7 @@ for fname in args:
     chg_mw = [[0 for i in range(16)] for j in range(len(pat))]
     chg_bank = [[0 for i in range(16)] for j in range(len(pat))]
     chg_prog = [[0 for i in range(16)] for j in range(len(pat))]
+    chg_vol = [[0 for i in range(16)] for j in range(len(pat))]
     ev_cnts = [[0 for i in range(16)] for j in range(len(pat))]
     tnames = [''] * len(pat)
     progs = set([0])
@@ -277,6 +287,14 @@ for fname in args:
                 elif ev.control == 33:  # ModWheel -- LSB
                     cur_mw[tidx][ev.channel] = (0x3F80 & cur_mw[tidx][ev.channel]) | ev.value
                     chg_mw[tidx][ev.channel] += 1
+                elif ev.control == 7:  # Volume -- MSB
+                    lvtime, lvol = sorted(vol_at[tidx][ev.channel].items(), key = lambda pair: pair[0])[-1]
+                    vol_at[tidx][ev.channel][abstime] = (0x3F & lvol) | (ev.value << 7)
+                    chg_vol[tidx][ev.channel] += 1
+                elif ev.control == 39:  # Volume -- LSB
+                    lvtime, lvol = sorted(vol_at[tidx][ev.channel].items(), key = lambda pair: pair[0])[-1]
+                    vol_at[tidx][ev.channel][abstime] = (0x3F80 & lvol) | ev.value
+                    chg_vol[tidx][ev.channel] += 1
                 events.append(MergeEvent(ev, tidx, abstime, cur_bank[tidx][ev.channel], cur_prog[tidx][ev.channel], cur_mw[tidx][ev.channel]))
                 ev_cnts[tidx][ev.channel] += 1
             elif isinstance(ev, midi.MetaEventWithText):
@@ -287,11 +305,10 @@ for fname in args:
                 events.append(MergeEvent(ev, tidx, abstime, cur_bank[tidx][ev.channel], cur_prog[tidx][ev.channel], cur_mw[tidx][ev.channel]))
                 ev_cnts[tidx][ev.channel] += 1
 
-    if options.verbose:
-        print 'Track name, event count, final banks, bank changes, final programs, program changes, final modwheel, modwheel changes:'
-        for tidx, tname in enumerate(tnames):
-            print tidx, ':', tname, ',', ','.join(map(str, ev_cnts[tidx])), ',', ','.join(map(str, cur_bank[tidx])), ',', ','.join(map(str, chg_bank[tidx])), ',', ','.join(map(str, cur_prog[tidx])), ',', ','.join(map(str, chg_prog[tidx])), ',', ','.join(map(str, cur_mw[tidx])), ',', ','.join(map(str, chg_mw[tidx]))
-        print 'All programs observed:', progs
+    print 'Track name, event count, final banks, bank changes, final programs, program changes, final modwheel, modwheel changes, volume changes:'
+    for tidx, tname in enumerate(tnames):
+        print tidx, ':', tname, ',', ','.join(map(str, ev_cnts[tidx])), ',', ','.join(map(str, cur_bank[tidx])), ',', ','.join(map(str, chg_bank[tidx])), ',', ','.join(map(str, cur_prog[tidx])), ',', ','.join(map(str, chg_prog[tidx])), ',', ','.join(map(str, cur_mw[tidx])), ',', ','.join(map(str, chg_mw[tidx])), ',', ','.join(map(str, chg_vol[tidx]))
+    print 'All programs observed:', progs
 
     print 'Sorting events...'
 
@@ -481,13 +498,15 @@ for fname in args:
                         realamp = dev.ampl
                         mwamp = float(dev.modwheel) / 0x3FFF
                         dt = 0.0
+                        origtime = dev.abstime
                         events = []
                         while dt < dev.duration:
+                            dev.abstime = origtime + dt
                             if options.modcont:
-                                t = dev.abstime + dt
+                                t = origtime
                             else:
                                 t = dt
-                            events.append(DurationEvent(dev, realpitch + mwamp * options.modfdev * math.sin(2 * math.pi * options.modffreq * t), realamp + mwamp * options.modadev * (math.sin(2 * math.pi * options.modafreq * t) - 1.0) / 2.0, min(dt, dev.duration - dt), dev.modwheel))
+                            events.append(DurationEvent(dev, realpitch + mwamp * options.modfdev * math.sin(2 * math.pi * options.modffreq * t), realamp + mwamp * options.modadev * (math.sin(2 * math.pi * options.modafreq * t) - 1.0) / 2.0, min(options.modres, dev.duration - dt), dev.modwheel))
                             dt += options.modres
                         ns.history[i:i+1] = events
                         i += len(events)
@@ -501,6 +520,95 @@ for fname in args:
                         i += 1
         print '...resolved', ev_cnt, 'events'
 
+    if options.stringres:
+        print 'Resolving string models...'
+        st_cnt = sum(sum(len(ns.history) for ns in group.streams) for group in notegroups)
+        in_cnt = 0
+        ex_cnt = 0
+        ev_cnt = 0
+        dev_grps = []
+        for group in notegroups:
+            for ns in group.streams:
+                i = 0
+                while i < len(ns.history):
+                    dev = ns.history[i]
+                    ntime = float('inf')
+                    if i + 1 < len(ns.history):
+                        ntime = ns.history[i+1].abstime
+                    dt = 0.0
+                    ampf = 1.0
+                    origtime = dev.abstime
+                    events = []
+                    while dt < dev.duration and ampf * dev.ampl >= options.stringthres:
+                        dev.abstime = origtime + dt
+                        events.append(DurationEvent(dev, dev.pitch, ampf * dev.ampl, min(options.stringres, dev.duration - dt), dev.modwheel))
+                        if len(events) > options.stringmax:
+                            print 'WARNING: Exceeded maximum string model events for event', i
+                            if options.verbose:
+                                print 'Final ampf', ampf, 'dt', dt
+                            break
+                        ampf *= options.stringrateon ** options.stringres
+                        dt += options.stringres
+                        in_cnt += 1
+                    dt = dev.duration
+                    while ampf * dev.ampl >= options.stringthres:
+                        dev.abstime = origtime + dt
+                        events.append(DurationEvent(dev, dev.pitch, ampf * dev.ampl, options.stringres, dev.modwheel))
+                        if len(events) > options.stringmax:
+                            print 'WARNING: Exceeded maximum string model events for event', i
+                            if options.verbose:
+                                print 'Final ampf', ampf, 'dt', dt
+                            break
+                        ampf *= options.stringrateoff ** options.stringres
+                        dt += options.stringres
+                        ex_cnt += 1
+                    if events:
+                        for j in xrange(len(events) - 1):
+                            cur, next = events[j], events[j + 1]
+                            if abs(cur.abstime + cur.duration - next.abstime) > options.epsilon:
+                                print 'WARNING: String model events cur: ', cur, 'next:', next, 'have gap/overrun of', next.abstime - (cur.abstime + cur.duration)
+                        dev_grps.append(events)
+                    else:
+                        print 'WARNING: Event', i, 'note', dev, ': No events?'
+                    if options.verbose:
+                        print 'Event', i, 'note', dev, 'in group', group.name, 'resolved to', len(events), 'events'
+                        if options.debug:
+                            for ev in events:
+                                print '\t', ev
+                    i += 1
+                    ev_cnt += len(events)
+        print '...resolved', ev_cnt, 'events (+', ev_cnt - st_cnt, ',', in_cnt, 'inside', ex_cnt, 'extra), resorting streams...'
+        for group in notegroups:
+            group.streams = []
+
+        dev_grps.sort(key = lambda evg: evg[0].abstime)
+        for devgr in dev_grps:
+            dev = devgr[0]
+            for group in notegroups:
+                if group.filter(dev):
+                    grp = group
+                    break
+            else:
+                grp = NSGroup()
+                notegroups.append(grp)
+            for ns in grp.streams:
+                if not ns.history:
+                    ns.history.extend(devgr)
+                    break
+                last = ns.history[-1]
+                if dev.abstime >= last.abstime + last.duration - 1e-3:
+                    ns.history.extend(devgr)
+                    break
+            else:
+                ns = NoteStream()
+                grp.streams.append(ns)
+                ns.history.extend(devgr)
+        scnt = 0
+        for group in notegroups:
+            for ns in group.streams:
+                scnt += 1
+        print 'Final sort:', len(notegroups), 'groups with', scnt, 'streams'
+
     if not options.keepempty:
         print 'Culling empty events...'
         ev_cnt = 0
@@ -520,6 +628,33 @@ for fname in args:
         for group in notegroups:
             print ('<anonymous>' if group.name is None else group.name), '<=', '(', len(group.streams), 'streams)'
 
+    print 'Final volume resolution...'
+    for group in notegroups:
+        for ns in group.streams:
+            for ev in ns.history:
+                t, vol = sorted(filter(lambda pair: pair[0] <= ev.abstime, vol_at[ev.tidx][ev.ev.channel].items()), key=lambda pair: pair[0])[-1]
+                ev.ampl *= (float(vol) / 0x3FFF) ** options.vol_pow
+
+    print 'Checking consistency...'
+    for group in notegroups:
+        if options.verbose:
+            print 'Group', '<None>' if group.name is None else group.name, 'with', len(group.streams), 'streams...',
+        ecnt = 0
+        for ns in group.streams:
+            for i in xrange(len(ns.history) - 1):
+                cur, next = ns.history[i], ns.history[i + 1]
+                if cur.abstime + cur.duration > next.abstime + options.epsilon:
+                    print 'WARNING: event', i, 'collides with next event (@', cur.abstime, '+', cur.duration, 'next @', next.abstime, ';', next.abstime - (cur.abstime + cur.duration), 'overlap)'
+                    ecnt += 1
+                if cur.abstime > next.abstime:
+                    print 'WARNING: event', i + 1, 'out of sort order (@', cur.abstime, 'next @', next.abstime, ';', cur.abstime - next.abstime, 'underlap)'
+                    ecnt += 1
+        if options.verbose:
+            if ecnt > 0:
+                print '...', ecnt, 'errors occured'
+            else:
+                print 'ok'
+
     print 'Generated %d streams in %d groups'%(sum(map(lambda x: len(x.streams), notegroups)), len(notegroups))
     print 'Playtime:', lastabstime, 'seconds'
 
@@ -557,7 +692,12 @@ for fname in args:
 
     ivtext = ET.SubElement(ivstreams, 'stream', type='text')
     for tev in textstream:
-        ivev = ET.SubElement(ivtext, 'text', time=str(tev.abstime), type=type(tev.ev).__name__, text=tev.ev.text)
+        text = tev.ev.text
+        # XXX Codec woes and general ET silliness
+        text = text.replace('\0', '')
+        #text = text.decode('latin_1')
+        #text = text.encode('ascii', 'replace')
+        ivev = ET.SubElement(ivtext, 'text', time=str(tev.abstime), type=type(tev.ev).__name__, text=text)
 
     ivaux = ET.SubElement(ivstreams, 'stream')
     ivaux.set('type', 'aux')
@@ -571,4 +711,5 @@ for fname in args:
         ivev.set('data', repr(fw.encode_midi_event(mev.ev)))
 
     print 'Done.'
-    open(os.path.splitext(os.path.basename(fname))[0]+'.iv', 'w').write(ET.tostring(iv))
+    txt = ET.tostring(iv, 'UTF-8')
+    open(os.path.splitext(os.path.basename(fname))[0]+'.iv', 'wb').write(txt)
diff --git a/packet.py b/packet.py
index 414ed73..1291601 100644
--- a/packet.py
+++ b/packet.py
@@ -27,7 +27,7 @@ class CMD:
         PCM = 5 # 16 samples, encoded S16_LE
 
 def itos(i):
-    return struct.pack('>L', i)
+    return struct.pack('>L', i).rstrip('\0')
 
 def stoi(s):
-    return struct.unpack('>L', s)[0]
+    return struct.unpack('>L', s.ljust(4, '\0'))[0]
diff --git a/shiv.py b/shiv.py
index ac6e2b1..051d175 100644
--- a/shiv.py
+++ b/shiv.py
@@ -8,6 +8,7 @@ import math
 parser = optparse.OptionParser()
 parser.add_option('-n', '--number', dest='number', action='store_true', help='Show number of tracks')
 parser.add_option('-g', '--groups', dest='groups', action='store_true', help='Show group names')
+parser.add_option('-G', '--group', dest='group', action='append', help='Only compute for this group (may be specified multiple times)')
 parser.add_option('-N', '--notes', dest='notes', action='store_true', help='Show number of notes')
 parser.add_option('-M', '--notes-stream', dest='notes_stream', action='store_true', help='Show notes per stream')
 parser.add_option('-m', '--meta', dest='meta', action='store_true', help='Show meta track information')
@@ -23,8 +24,9 @@ parser.add_option('-x', '--aux', dest='aux', action='store_true', help='Show inf
 
 parser.add_option('-a', '--almost-all', dest='almost_all', action='store_true', help='Show useful information')
 parser.add_option('-A', '--all', dest='all', action='store_true', help='Show everything')
+parser.add_option('-t', '--total', dest='total', action='store_true', help='Make cross-file totals')
 
-parser.set_defaults(height=20)
+parser.set_defaults(height=20, group=[])
 
 options, args = parser.parse_args()
 
@@ -65,6 +67,7 @@ else:
 def show_hist(values, height=None):
     if not values:
         print '{empty histogram}'
+        return
     if height is None:
         height = options.height
     xs, ys = values.keys(), values.values()
@@ -85,16 +88,29 @@ def show_hist(values, height=None):
         print COL.YELLOW + '\t   ' + ''.join([s[i] if len(s) > i else ' ' for s in xcs]) + COL.NONE
     print
 
+if options.total:
+    tot_note_cnt = 0
+    max_note_cnt = 0
+    tot_pitches = {}
+    tot_velocities = {}
+    tot_dur = 0
+    max_dur = 0
+    tot_streams = 0
+    max_streams = 0
+    tot_notestreams = 0
+    max_notestreams = 0
+    tot_groups = {}
+
 for fname in args:
+    print
+    print 'File :', fname
     try:
         iv = ET.parse(fname).getroot()
-    except IOError:
+    except Exception:
         import traceback
         traceback.print_exc()
         print 'Bad file :', fname, ', skipping...'
         continue
-    print
-    print 'File :', fname
     print '\t<computing...>'
 
     if options.meta:
@@ -115,10 +131,19 @@ for fname in args:
     streams = iv.findall('./streams/stream')
     notestreams = [s for s in streams if s.get('type') == 'ns']
     auxstreams = [s for s in streams if s.get('type') == 'aux']
+    if options.group:
+        print 'NOTE: Restricting results to groups', options.group, 'as requested'
+        notestreams = [ns for ns in notestreams if ns.get('group', '<anonymous>') in options.group]
+
     if options.number:
         print 'Stream count:'
         print '\tNotestreams:', len(notestreams)
         print '\tTotal:', len(streams)
+        if options.total:
+            tot_streams += len(streams)
+            max_streams = max(max_streams, len(streams))
+            tot_notestreams += len(notestreams)
+            max_notestreams = max(max_notestreams, len(notestreams))
 
     if not (options.groups or options.notes or options.histogram or options.histogram_tracks or options.vel_hist or options.vel_hist_tracks or options.duration or options.duty_cycle or options.aux):
         continue
@@ -128,6 +153,8 @@ for fname in args:
         for s in notestreams:
             group = s.get('group', '<anonymous>')
             groups[group] = groups.get(group, 0) + 1
+            if options.total:
+                tot_groups[group] = tot_groups.get(group, 0) + 1
         print 'Groups:'
         for name, cnt in groups.iteritems():
             print '\t{} ({} streams)'.format(name, cnt)
@@ -180,14 +207,20 @@ for fname in args:
             dur = float(note.get('dur'))
             if options.notes:
                 note_cnt += 1
+                if options.total:
+                    tot_note_cnt += 1
             if options.notes_stream:
                 notes_stream[sidx] += 1
             if options.histogram:
                 pitches[pitch] = pitches.get(pitch, 0) + 1
+                if options.total:
+                    tot_pitches[pitch] = tot_pitches.get(pitch, 0) + 1
             if options.histogram_tracks:
                 pitch_tracks[sidx][pitch] = pitch_tracks[sidx].get(pitch, 0) + 1
             if options.vel_hist:
                 velocities[vel] = velocities.get(vel, 0) + 1
+                if options.total:
+                    tot_velocities[vel] = tot_velocities.get(vel, 0) + 1
             if options.vel_hist_tracks:
                 velocities_tracks[sidx][vel] = velocities_tracks[sidx].get(vel, 0) + 1
             if (options.duration or options.duty_cycle) and time + dur > max_dur:
@@ -195,6 +228,9 @@ for fname in args:
             if options.duty_cycle:
                 cum_dur[sidx] += dur
 
+    if options.notes and options.total:
+        max_note_cnt = max(max_note_cnt, note_cnt)
+
     if options.histogram_tracks:
         for sidx, hist in enumerate(pitch_tracks):
             print 'Stream {} (group {}) pitch histogram:'.format(sidx, notestreams[sidx].get('group', '<anonymous>'))
@@ -219,3 +255,27 @@ for fname in args:
         show_hist(velocities)
     if options.duration:
         print 'Playing duration: {}'.format(max_dur)
+
+if options.total:
+    print 'Totals:'
+    if options.number:
+        print '\tTotal streams:', tot_streams
+        print '\tMax streams:', max_streams
+        print '\tTotal notestreams:', tot_notestreams
+        print '\tMax notestreams:', max_notestreams
+        print
+    if options.notes:
+        print '\tTotal notes:', tot_note_cnt
+        print '\tMax notes:', max_note_cnt
+        print
+    if options.groups:
+        print '\tGroups:'
+        for grp, cnt in tot_groups.iteritems():
+            print '\t\t', grp, ':', cnt
+        print
+    if options.histogram:
+        print 'Overall pitch histogram:'
+        show_hist(tot_pitches)
+    if options.vel_hist:
+        print 'Overall velocity histogram:'
+        show_hist(tot_velocities)