Merge pull request #105 from Ridderrasmus/recent-changes

Downstream commits from net7

src/Audio/AudioData.cs

@@ -10,19 +10,19 @@ public class AudioData
         public ALFormat format;
         public double amplitude;
         public VoiceLevel voiceLevel;
+        public string codec;
 
         public AudioData() { }
 
-        public static AudioData FromPacket(AudioPacket audioPacket, IAudioCodec codec = null)
+        public static AudioData FromPacket(AudioPacket audioPacket)
         {
-            var data = codec?.Decode(audioPacket.AudioData) ?? audioPacket.AudioData;
-
             return new AudioData()
             {
-                data = data,
+                data = audioPacket.AudioData,
                 frequency = audioPacket.Frequency,
                 format = audioPacket.Format,
                 voiceLevel = audioPacket.VoiceLevel,
+                codec = audioPacket.Codec
             };
         }
     }

src/Audio/Codecs/IAudioCodec.cs

@@ -2,8 +2,11 @@
 {
     public interface IAudioCodec
     {
+        public string Name { get; }
+        public int SampleRate { get; }
+        public int Channels { get; }
+        public int FrameSize { get; }
         public byte[] Encode(short[] pcmData);
         public byte[] Decode(byte[] encodedData);
-        public int GetFrameSize();
     }
 }

src/Audio/Codecs/OpusCodec.cs

@@ -8,22 +8,24 @@ namespace RPVoiceChat.Audio
 {
     public class OpusCodec : IAudioCodec
     {
+        public const string _Name = "Opus";
+        public string Name { get; } = _Name;
+        public int SampleRate { get; }
+        public int Channels { get; }
+        public int FrameSize { get; }
         private OpusEncoder encoder;
         private OpusDecoder decoder;
-        private int sampleRate;
-        private int channels;
-        private int frameSize;
 
         public OpusCodec(int frequency, int channelCount)
         {
-            sampleRate = frequency;
-            channels = channelCount;
-            frameSize = sampleRate / 100 * channels; // 10ms frame window
+            SampleRate = frequency;
+            Channels = channelCount;
+            FrameSize = SampleRate / 100 * Channels; // 10ms frame window
 
-            encoder = new OpusEncoder(sampleRate, channels, OpusApplication.OPUS_APPLICATION_VOIP);
-            decoder = new OpusDecoder(sampleRate, channels);
+            encoder = new OpusEncoder(SampleRate, Channels, OpusApplication.OPUS_APPLICATION_VOIP);
+            decoder = new OpusDecoder(SampleRate, Channels);
 
-            encoder.Bitrate = 40000;
+            encoder.Bitrate = 40 * 1024;
             encoder.Complexity = 10;
             encoder.SignalType = OpusSignal.OPUS_SIGNAL_VOICE;
             encoder.ForceMode = OpusMode.MODE_SILK_ONLY;
@@ -35,18 +37,17 @@ public OpusCodec(int frequency, int channelCount)
         public byte[] Encode(short[] pcmData)
         {
             const int maxPacketSize = 1276;
+            byte[] encodedBuffer = new byte[maxPacketSize];
             var encoded = new MemoryStream();
-
             try
             {
-                for (var pcmOffset = 0; pcmOffset < pcmData.Length; pcmOffset += frameSize)
+                for (var pcmOffset = 0; pcmOffset < pcmData.Length; pcmOffset += FrameSize)
                 {
-                    byte[] encodedData = new byte[maxPacketSize];
-                    int encodedLength = encoder.Encode(pcmData, pcmOffset, frameSize, encodedData, 0, maxPacketSize);
+                    int encodedLength = encoder.Encode(pcmData, pcmOffset, FrameSize, encodedBuffer, 0, maxPacketSize);
                     byte[] packetSize = BitConverter.GetBytes(encodedLength);
 
                     encoded.Write(packetSize, 0, packetSize.Length);
-                    encoded.Write(encodedData, 0, encodedLength);
+                    encoded.Write(encodedBuffer, 0, encodedLength);
                 }
             }
             catch (Exception e)
@@ -59,6 +60,7 @@ public byte[] Encode(short[] pcmData)
 
         public byte[] Decode(byte[] encodedData)
         {
+            short[] decodedBuffer = new short[FrameSize];
             var decoded = new MemoryStream();
             var stream = new MemoryStream(encodedData);
             using (var reader = new BinaryReader(stream))
@@ -70,9 +72,8 @@ public byte[] Decode(byte[] encodedData)
                         int packetSize = reader.ReadInt32();
                         byte[] encodedPacket = reader.ReadBytes(packetSize);
 
-                        short[] decodedData = new short[frameSize];
-                        int decodedLength = decoder.Decode(encodedPacket, 0, packetSize, decodedData, 0, frameSize, false);
-                        byte[] decodedPacket = ToBytes(decodedData, 0, decodedLength);
+                        int decodedLength = decoder.Decode(encodedPacket, 0, packetSize, decodedBuffer, 0, FrameSize, false);
+                        byte[] decodedPacket = ShortsToBytes(decodedBuffer, 0, decodedLength);
 
                         decoded.Write(decodedPacket, 0, decodedPacket.Length);
                     }
@@ -86,17 +87,11 @@ public byte[] Decode(byte[] encodedData)
             return decoded.ToArray();
         }
 
-        public int GetFrameSize()
-        {
-            return frameSize;
-        }
-
-        private byte[] ToBytes(short[] audio, int offset, int length)
+        public static byte[] ShortsToBytes(short[] audio, int offset, int length)
         {
-            int typeSize = sizeof(short);
-            byte[] byteBuffer = new byte[length * typeSize];
-            int bytesToCopy = (length - offset) * typeSize;
-            Buffer.BlockCopy(audio, offset, byteBuffer, offset * typeSize, bytesToCopy);
+            byte[] byteBuffer = new byte[length * sizeof(short)];
+            int bytesToCopy = (length - offset) * sizeof(short);
+            Buffer.BlockCopy(audio, offset, byteBuffer, offset * sizeof(short), bytesToCopy);
 
             return byteBuffer;
         }

src/Audio/Input/MicrophoneManager.cs

@@ -121,7 +121,7 @@ private void UpdateCaptureAudioSamples()
             if (clientEntity == null || capture == null) return;
 
             int samplesAvailable = capture.AvailableSamples;
-            int frameSize = codec.GetFrameSize();
+            int frameSize = codec.FrameSize;
             int samplesToRead = samplesAvailable - samplesAvailable % frameSize;
             if (samplesToRead <= 0) return;
             int bufferLength = samplesToRead * SampleToByte * InputChannelCount;
@@ -173,15 +173,20 @@ private AudioData ProcessAudio(byte[] rawSamples)
 
             var amplitude = Math.Sqrt(sampleSquareSum / pcmCount);
 
-            byte[] opusEncodedAudio = codec.Encode(pcms);
+            byte[] audio;
+            bool shouldEncode = WorldConfig.GetBool("encode-audio");
+            if (shouldEncode) audio = codec.Encode(pcms);
+            else audio = OpusCodec.ShortsToBytes(pcms, 0, pcms.Length);
+            string codecName = shouldEncode ? codec.Name : "None";
 
             return new AudioData()
             {
-                data = opusEncodedAudio,
+                data = audio,
                 frequency = Frequency,
                 format = OutputFormat,
                 amplitude = amplitude,
                 voiceLevel = voiceLevel,
+                codec = codecName
             };
         }
 

src/Audio/Output/AudioOutputManager.cs

@@ -1,4 +1,4 @@
-using OpenTK.Audio.OpenAL;
+using OpenTK.Audio.OpenAL;
 using RPVoiceChat.DB;
 using RPVoiceChat.Networking;
 using RPVoiceChat.Utils;
@@ -67,36 +67,27 @@ public void HandleAudioPacket(AudioPacket packet)
                 return;
             }
 
-            PlayerAudioSource source;
-            string playerId = packet.PlayerId;
-
-            if (!playerSources.TryGetValue(playerId, out source) || source.IsDisposed)
+            PlayerAudioSource source = GetOrCreatePlayerSource(packet.PlayerId);
+            if (source == null)
             {
-                var player = capi.World.PlayerByUid(playerId);
-                if (player == null)
-                {
-                    Logger.client.Error($"Could not find player for playerId {playerId}");
-                    return;
-                }
-
-                source = CreatePlayerSource(player);
+                Logger.client.Debug("Unable to resolve player ID into player source, dropping packet");
+                return;
             }
 
             HandleAudioPacket(packet, source);
         }
 
         public void HandleAudioPacket(AudioPacket packet, PlayerAudioSource source)
         {
+            string codec = packet.Codec;
             int frequency = packet.Frequency;
             int channels = AudioUtils.ChannelsPerFormat(packet.Format);
+            AudioData audioData = AudioData.FromPacket(packet);
 
-            IAudioCodec codec = source.GetOrCreateAudioCodec(frequency, channels);
-            AudioData audioData = AudioData.FromPacket(packet, codec);
-
-            // Update the voice level if it has changed
             if (source.voiceLevel != packet.VoiceLevel)
                 source.UpdateVoiceLevel(packet.VoiceLevel);
             source.UpdatePlayer();
+            source.UpdateAudioFormat(codec, frequency, channels);
             source.EnqueueAudio(audioData, packet.SequenceNumber);
         }
 
@@ -118,6 +109,18 @@ private void ClientLoaded()
             localPlayerAudioSource.StartPlaying();
         }
 
+        private PlayerAudioSource GetOrCreatePlayerSource(string playerId)
+        {
+            PlayerAudioSource source;
+            if (playerSources.TryGetValue(playerId, out source) && !source.IsDisposed)
+                return source;
+
+            var player = capi.World.PlayerByUid(playerId);
+            if (player == null) return null;
+
+            return CreatePlayerSource(player);
+        }
+
         private PlayerAudioSource CreatePlayerSource(IPlayer player)
         {
             var source = new PlayerAudioSource(player, capi, clientSettings, effectsExtension);

src/Audio/Output/PlayerAudioSource.cs

@@ -4,7 +4,6 @@
 using RPVoiceChat.Gui;
 using RPVoiceChat.Utils;
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using Vintagestory.API.Client;
@@ -21,10 +20,12 @@ public class PlayerAudioSource : IDisposable
         private const int BufferCount = 20;
         private int source;
         private CircularAudioBuffer buffer;
-        private SortedList orderingQueue = SortedList.Synchronized(new SortedList());
+        private SortedList<long, AudioData> orderingQueue = new SortedList<long, AudioData>();
+        private object ordering_queue_lock = new object();
+        private object dequeue_audio_lock = new object();
         private int orderingDelay = 100;
         private long lastAudioSequenceNumber = -1;
-        private Dictionary<string, IAudioCodec> codecs = new Dictionary<string, IAudioCodec>();
+        private IAudioCodec codec;
         private FilterLowpass lowpassFilter;
         private EffectsExtension effectsExtension;
 
@@ -77,18 +78,14 @@ public void UpdateVoiceLevel(VoiceLevel voiceLevel)
             OALW.Source(source, ALSourcef.RolloffFactor, rolloffFactor);
         }
 
-        public IAudioCodec GetOrCreateAudioCodec(int frequency, int channels)
+        public void UpdateAudioFormat(string codecName, int frequency, int channels)
         {
-            string codecSettings = $"{frequency}:{channels}";
-            IAudioCodec codec;
-
-            if (!codecs.TryGetValue(codecSettings, out codec))
-            {
-                codec = new OpusCodec(frequency, channels);
-                codecs.Add(codecSettings, codec);
-            }
-
-            return codec;
+            if (codec?.Name == codecName && codec?.SampleRate == frequency && codec?.Channels == channels) return;
+            codec = codecName switch {
+                OpusCodec._Name => new OpusCodec(frequency, channels),
+                "None" => null,
+                _ => null
+            };
         }
 
         public void UpdatePlayer()
@@ -210,42 +207,47 @@ private Vec3f GetVelocity(EntityPos speakerPos)
 
         public void EnqueueAudio(AudioData audio, long sequenceNumber)
         {
-            if (orderingQueue.ContainsKey(sequenceNumber))
-            {
-                Logger.client.Debug("Audio sequence already received, skipping enqueueing");
-                return;
-            }
-
-            if (lastAudioSequenceNumber > sequenceNumber)
+            lock (ordering_queue_lock)
             {
-                Logger.client.Debug("Audio sequence arrived too late, skipping enqueueing");
-                return;
+                if (orderingQueue.ContainsKey(sequenceNumber))
+                {
+                    Logger.client.VerboseDebug($"Audio sequence {sequenceNumber} already received, skipping enqueueing");
+                    return;
+                }
+
+                if (lastAudioSequenceNumber >= sequenceNumber)
+                {
+                    Logger.client.VerboseDebug($"Audio sequence {sequenceNumber} arrived too late, skipping enqueueing");
+                    return;
+                }
+
+                orderingQueue.Add(sequenceNumber, audio);
             }
 
-            orderingQueue.Add(sequenceNumber, audio);
             DequeueAudio();
         }
 
         public async void DequeueAudio()
         {
-            AudioData audio;
-
             await Task.Delay(orderingDelay);
 
-            lock (orderingQueue.SyncRoot)
-            {
-                audio = orderingQueue.GetByIndex(0) as AudioData;
-                lastAudioSequenceNumber = (long)orderingQueue.GetKey(0);
-                orderingQueue.RemoveAt(0);
-            }
-
-            buffer.QueueAudio(audio.data, audio.format, audio.frequency);
-
-            // The source can stop playing if it finishes everything in queue
-            var state = OALW.GetSourceState(source);
-            if (state != ALSourceState.Playing)
+            lock (dequeue_audio_lock)
             {
-                StartPlaying();
+                AudioData audio;
+                lock (ordering_queue_lock)
+                {
+                    lastAudioSequenceNumber = orderingQueue.Keys[0];
+                    audio = orderingQueue[lastAudioSequenceNumber];
+                    orderingQueue.RemoveAt(0);
+                }
+
+                if (codec != null) audio.data = codec.Decode(audio.data);
+                buffer.QueueAudio(audio.data, audio.format, audio.frequency);
+
+                // The source can stop playing if it finishes everything in queue
+                var state = OALW.GetSourceState(source);
+                if (state != ALSourceState.Playing)
+                    StartPlaying();
             }
         }
 

src/Networking/Packets/AudioPacket.cs

@@ -14,6 +14,7 @@ public class AudioPacket : NetworkPacket
         public int Frequency { get; set; }
         public ALFormat Format { get; set; }
         public long SequenceNumber { get; set; }
+        public string Codec { get; set; }
         protected override PacketType Code { get => PacketType.Audio; }
 
         public AudioPacket() { }
@@ -27,6 +28,7 @@ public AudioPacket(string playerId, AudioData audioData, long sequenceNumber)
             Frequency = audioData.frequency;
             Format = audioData.format;
             SequenceNumber = sequenceNumber;
+            Codec = audioData.codec;
         }
     }
 }