Protobuf based query builders

Refactor output of movement into protobuf based

WowPacketParser.Proto/PacketStructures/structures.proto

@@ -344,15 +344,6 @@ enum UniversalSplineFlag {
   Steering            = 0x40000000;
 }
 
-enum CreatureMovementFlags {
-  NormalPathfinding       = 0; // pathfinding
-  ExactPath               = 1; // sent fully
-  ExactPathFlying         = 2; // sent fully + flying flag
-  ExactPathFlyingCyclic   = 3; // sent fully + flying flag + cyclic flag
-  ExactPathAndJump        = 4; // sent fully + parabolic movement at the end
-  CombatMovement          = 100;
-}
-
 message PacketMonsterMove {
   UniversalGuid mover = 1;
   Vec3 position = 2;
@@ -371,8 +362,7 @@ message PacketMonsterMove {
   repeated Vec3 points = 13;
   repeated Vec3 packedPoints = 14;
   optional SplineJump jump = 15;
-  optional CreatureMovementFlags creatureMovementFlags = 16;
-  bool creationSpline = 17;
+  bool creationSpline = 16;
 }
 
 message SplineJump {

WowPacketParser/Loading/SniffFile.cs

@@ -83,18 +83,19 @@ private string FileName
             }
         }
 
-        public void ProcessFile()
+        public Packets ProcessFile()
         {
 
             try
             {
-                ProcessFileImpl();
+                return ProcessFileImpl();
             }
             catch (Exception ex)
             {
                 Trace.WriteLine(_logPrefix + " " + ex.GetType());
                 Trace.WriteLine(_logPrefix + " " + ex.Message);
                 Trace.WriteLine(_logPrefix + " " + ex.StackTrace);
+                return new Packets();
             }
             finally
             {
@@ -106,7 +107,7 @@ public void ProcessFile()
             }
         }
 
-        private void ProcessFileImpl()
+        private Packets ProcessFileImpl()
         {
             if (_compression != FileCompression.None)
                 _tempName = Decompress();
@@ -184,9 +185,9 @@ private void ProcessFileImpl()
 
                     var written = false;
 
+                    Packets packets = new() { Version = StructureVersion.ProtobufStructureVersion, DumpType = (uint)Settings.DumpFormat };
                     using (var writer = (Settings.DumpFormatWithTextToFile() ? new StreamWriter(outFileName, true) : null))
                     {
-                        Packets packets = new() { Version = StructureVersion.ProtobufStructureVersion, DumpType = (uint)Settings.DumpFormat };
                         var firstRead = true;
                         var firstWrite = true;
 
@@ -285,7 +286,7 @@ private void ProcessFileImpl()
                             // Close Writer, Stream - Dispose
                             packet.ClosePacket();
 
-                            if (_dumpFormat.IsUniversalProtobufType())
+                            if (_dumpFormat.IsUniversalProtobufType() || Settings.SQLOutputFlag != 0 || HotfixSettings.Instance.ShouldLog())
                                 packets.Packets_.Add(packet.Holder);
                         }, threadCount);
 
@@ -316,14 +317,14 @@ private void ProcessFileImpl()
                     Trace.WriteLine($"{_logPrefix}: {_stats}");
 
                     if (Settings.SQLOutputFlag != 0 || HotfixSettings.Instance.ShouldLog())
-                        WriteSQLs();
+                        WriteSQLs(packets);
 
                     if (Settings.LogPacketErrors)
                         WritePacketErrors();
 
                     GC.Collect(); // Force a GC collect after parsing a file. It seems to help.
 
-                    break;
+                    return packets;
                 }
                 case DumpFormatType.Pkt:
                 {
@@ -478,6 +479,8 @@ private void ProcessFileImpl()
                     break;
                 }
             }
+
+            return new Packets();
         }
 
         public static string GetHeader(string fileName)
@@ -560,14 +563,14 @@ private void BinaryDump(string fileName, ICollection<Packet> packets)
             BinaryPacketWriter.Write(SniffType.Pkt, fileName, Encoding.ASCII, packets);
         }
 
-        private void WriteSQLs()
+        private void WriteSQLs(Packets packets)
         {
             var sqlFileName = string.IsNullOrWhiteSpace(Settings.SQLFileName) ? $"{Utilities.FormattedDateTimeForFiles()}_{Path.GetFileName(FileName)}.sql" : Settings.SQLFileName;
 
             if (!string.IsNullOrWhiteSpace(Settings.SQLFileName))
                 return;
 
-            Builder.DumpSQL($"{_logPrefix}: Dumping sql", sqlFileName, GetHeader(FileName));
+            Builder.DumpSQL(new []{packets}, $"{_logPrefix}: Dumping sql", sqlFileName, GetHeader(FileName));
             Storage.ClearContainers();
         }
 

WowPacketParser/PacketStructures/ProtoExtensions.cs

@@ -256,5 +256,101 @@ public static CreateObjectType ToCreateObjectType(this UpdateTypeCataclysm updat
                 return CreateObjectType.Spawn;
             throw new ArgumentOutOfRangeException();
         }
+
+        public static ObjectType ToObjectType(this UniversalHighGuid highGuid)
+        {
+            switch (highGuid)
+            {
+                case UniversalHighGuid.Player:
+                    return ObjectType.Player;
+                case UniversalHighGuid.DynamicObject:
+                    return ObjectType.DynamicObject;
+                case UniversalHighGuid.Item:
+                    return ObjectType.Item;
+                case UniversalHighGuid.GameObject:
+                case UniversalHighGuid.Transport:
+                    return ObjectType.GameObject;
+                case UniversalHighGuid.Vehicle:
+                case UniversalHighGuid.Creature:
+                case UniversalHighGuid.Pet:
+                    return ObjectType.Unit;
+                case UniversalHighGuid.AreaTrigger:
+                    return ObjectType.AreaTrigger;
+                default:
+                    return ObjectType.Object;
+            }
+        }
+
+        public static string ToWowParserString(this UniversalGuid guid)
+        {
+            if (guid.KindCase == UniversalGuid.KindOneofCase.Guid128 && (guid.Guid128.High != 0 || guid.Guid128.Low != 0))
+            {
+                var guid128 = new WowGuid128(guid.Guid128.Low, guid.Guid128.High);
+                return guid128.ToString();
+            }
+            else if (guid.KindCase == UniversalGuid.KindOneofCase.Guid64 && (guid.Guid64.High != 0 || guid.Guid64.Low != 0))
+            {
+                var guid64 = new WowGuid64(guid.Guid64.Low);
+                return guid64.ToString();
+            }
+
+            return "Full: 0x0";
+        }
+
+        public static uint GetLow(this UniversalGuid guid)
+        {
+            if (guid.KindCase == UniversalGuid.KindOneofCase.Guid128)
+                return (uint)(guid.Guid128.Low & 0xFFFFFFFFFF);
+            else if (guid.KindCase == UniversalGuid.KindOneofCase.Guid64)
+            {
+                switch (guid.Type)
+                {
+                    case UniversalHighGuid.Player:
+                    case UniversalHighGuid.DynamicObject:
+                    case UniversalHighGuid.RaidGroup:
+                    case UniversalHighGuid.Item:
+                        return (uint)(guid.Guid64.Low & 0x000FFFFFFFFFFFFF);
+                    case UniversalHighGuid.GameObject:
+                    case UniversalHighGuid.Transport:
+                    //case HighGuidType.MOTransport: ??
+                    case UniversalHighGuid.Vehicle:
+                    case UniversalHighGuid.Creature:
+                    case UniversalHighGuid.Pet:
+                        return (uint)(guid.Guid64.Low & 0x00000000FFFFFFFFul);
+                }
+
+                return (uint)(guid.Guid64.Low & 0x00000000FFFFFFFFul);
+            }
+
+            return 0;
+        }
+
+        public static uint GetEntry(this UniversalGuid128 guid)
+        {
+            return (uint)((guid.High >> 6) & 0x7FFFFF);
+        }
+
+        public static byte GetSubType(this UniversalGuid128 guid) => (byte)(guid.High & 0x3F);
+
+        public static ushort GetRealmId(this UniversalGuid128 guid) => (ushort)((guid.High >> 42) & 0x1FFF);
+
+        public static uint GetServerId(this UniversalGuid128 guid) => (uint)((guid.Low >> 40) & 0xFFFFFF);
+
+        public static ushort GetMapId(this UniversalGuid128 guid) => (ushort)((guid.High >> 29) & 0x1FFF);
+
+        public static bool HasEntry(this UniversalGuid guid)
+        {
+            switch (guid.Type)
+            {
+                case UniversalHighGuid.Creature:
+                case UniversalHighGuid.GameObject:
+                case UniversalHighGuid.Pet:
+                case UniversalHighGuid.Vehicle:
+                case UniversalHighGuid.AreaTrigger:
+                    return true;
+                default:
+                    return false;
+            }
+        }
     }
 }

WowPacketParser/Parsing/Proto/BaseProtoQueryBuilder.cs

@@ -0,0 +1,26 @@
+using System.Collections.Generic;
+using WowPacketParser.Proto;
+using WowPacketParser.Proto.Processing;
+
+namespace WowPacketParser.Parsing.Proto;
+
+public abstract class BaseProtoQueryBuilder : PacketProcessor<VoidType>, IProtoQueryBuilder
+{
+    public abstract bool IsEnabled();
+
+    public string Process(IReadOnlyList<Packets> packetsList)
+    {
+        foreach (var sniffFile in packetsList)
+        {
+            // more complex logic is possible, i.e. clear state after each file
+            foreach (var packet in sniffFile.Packets_)
+            {
+                Process(packet);
+            }
+        }
+
+        return GenerateQuery();
+    }
+
+    protected abstract string GenerateQuery();
+}

WowPacketParser/Parsing/Proto/IProtoQueryBuilder.cs

@@ -0,0 +1,10 @@
+using System.Collections.Generic;
+using WowPacketParser.Proto;
+
+namespace WowPacketParser.Parsing.Proto;
+
+public interface IProtoQueryBuilder
+{
+    bool IsEnabled();
+    string Process(IReadOnlyList<Packets> packets);
+}

WowPacketParser/Parsing/Proto/VoidType.cs

@@ -0,0 +1,3 @@
+namespace WowPacketParser.Parsing.Proto;
+
+public struct VoidType { }

WowPacketParser/Program.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
 using System.IO;
@@ -9,6 +10,7 @@
 using WowPacketParser.Loading;
 using WowPacketParser.Misc;
 using WowPacketParser.Parsing.Parsers;
+using WowPacketParser.Proto;
 using WowPacketParser.SQL;
 
 namespace WowPacketParser
@@ -77,6 +79,8 @@ private static void Main(string[] args)
 
             SQLConnector.ReadDB();
 
+            List<Packets> parserPacketsList = new();
+
             var processStartTime = DateTime.Now;
             var count = 0;
             foreach (var file in files)
@@ -90,7 +94,7 @@ private static void Main(string[] args)
                 try
                 {
                     var sf = new SniffFile(file, Settings.DumpFormat, Tuple.Create(++count, files.Count));
-                    sf.ProcessFile();
+                    parserPacketsList.Add(sf.ProcessFile());
                 }
                 catch (IOException ex)
                 {
@@ -99,7 +103,7 @@ private static void Main(string[] args)
             }
 
             if (!string.IsNullOrWhiteSpace(Settings.SQLFileName) && Settings.DumpFormatWithSQL())
-                Builder.DumpSQL("Dumping global sql", Settings.SQLFileName, SniffFile.GetHeader("multi"));
+                Builder.DumpSQL(parserPacketsList, "Dumping global sql", Settings.SQLFileName, SniffFile.GetHeader("multi"));
 
             var processTime = DateTime.Now.Subtract(processStartTime);
             Trace.WriteLine($"Processing {files.Count} sniffs took { processTime.ToFormattedString() }.");