Well, I would like to do my own benchmarking system like spark in Minecraft (https://github.com/lucko/spark):
I'm using Harmony lib (https://github.com/pardeike/Harmony) which allows me to interact/modify methods and allows me to add a Prefix/Postfix on each call that will help me out with this stack.
The basic structure has something similar to (https://github.com/pardeike/Harmony/issues/355):
[HarmonyPatch]
class MyPatches
{
static IEnumerable<MethodBase> TargetMethods()
{
return AccessTools.GetTypesFromAssembly(Assembly.GetExecutingAssembly())
.SelectMany(type => type.GetMethods())
.Where(method => method.ReturnType != typeof(void) && method.Name.StartsWith("Do"));
}
static void Prefix(out Stopwatch __state, MethodBase __originalMethod)
{
__state = Stopwatch.StartNew();
// ...
}
static void Postfix(Stopwatch __state, MethodBase __originalMethod)
{
__state.Stop();
// ....
}
}
The problem here is that the __originalMethod doesn't take care if it was called from A or B.
So for example, we had patched string.Join method. And the we call from A or B, where A or B, is the full callstack of this method.
So first, we need to assign a ID to this call, and we need to create a Tree-based structure (which is hard to serialize later), from here (https://stackoverflow.com/a/36649069/3286975):
public class TreeModel : Tree<TreeModel>
{
public int ID { get; set; }
public TreeModel() { }
public TreeModel(TreeModel parent) : base(parent) { }
}
public class Tree<T> where T : Tree<T>
{
protected Tree() : this(null) { }
protected Tree(T parent)
{
Parent=parent;
Children=new List<T>();
if(parent!=null)
{
parent.Children.Add(this as T);
}
}
public T Parent { get; set; }
public List<T> Children { get; set; }
public bool IsRoot { get { return Parent==null; } }
public T Root { get { return IsRoot?this as T:Parent.Root; } }
public T RecursiveFind(Predicate<T> check)
{
if(check(this as T)) return this as T;
foreach(var item in Children)
{
var result=item.RecursiveFind(check);
if(result!=null)
{
return result;
}
}
return null;
}
}
Now, the thing is that we need to fill the Tree as long as we iterate all the method and instructions got from Harmony. Forget about Harmony for a second, I will explain only two facts about it.
The lib allows you first to get all patched methods through IEnumerable<MethodBase> TargetMethods() so, you have the Assembly X passed through reflection and filtered all methods that are allowed to be patched (some of them broke Unity, so I decided to skip methods from UnityEngine., UnityEditor. and System.* namespaces).
And we have also the ReadMethodBody method (https://harmony.pardeike.net/api/HarmonyLib.PatchProcessor.html#HarmonyLib_PatchProcessor_ReadMethodBody_System_Reflection_MethodBase_) from a given MethodBase it returns all IL stack instructions.
So we can start to iterate over and over in order to get all instructions and fill the entire tree. This is what I wrote last night:
internal static class BenchmarkEnumerator
{
internal static Dictionary<MethodBase, int> Mappings { get; } = new Dictionary<MethodBase, int>();
internal static Dictionary<int, TreeModel> TreeIDs { get; } = new Dictionary<int, TreeModel>();
internal static Dictionary<MethodBase, BenchmarkTreeModel> TreeMappings { get; } = new Dictionary<MethodBase, BenchmarkTreeModel>();
private static HashSet<int> IDUsed { get; } = new HashSet<int>();
public static int GetID(this MethodBase method)
{
return GetID(method, out _);
}
public static int GetID(this MethodBase method, out bool contains)
{
// A > X = X1
// B > X = X2
if (!Mappings.ContainsKey(method))
{
var id = Mappings.Count;
Mappings.Add(method, Mappings.Count);
IDUsed.Add(id);
contains = false;
return id;
}
contains = true;
return Mappings[method];
}
public static int GetFreeID()
{
int id;
Random rnd = new Random();
do
{
id = rnd.Next();
} while (IDUsed.Contains(id));
IDUsed.Add(id);
return id;
}
public static BenchmarkCall GetCall(int id)
{
return TreeIDs[id]?.Call;
}
public static BenchmarkCall GetCall(this MethodBase method)
{
return TreeIDs[Mappings[method]]?.Call;
}
}
The BenchmarkEnumerator class allow us to differentiate between A or B, but it doesn't care about the full hierarchy, only from the parent MethodBase itself, so I need to write something complex to take in care of the full call stack, which I said I have a problem to understand.
Then we have the TargetMethods:
private static IEnumerable<MethodBase> TargetMethods()
{
Model = new BenchmarkTreeModel();
var sw = Stopwatch.StartNew();
//int i = 0;
return Filter.GetTargetMethods(method =>
{
try
{
var instructions = PatchProcessor.ReadMethodBody(method);
var i = method.GetID(out var contains);
var tree = new TreeModel
{
ID = i
};
if (contains)
{
//var lastId = i;
i = GetFreeID();
tree.ID = i;
tree.FillMethodName($"{method.GetMethodSignature()}_{i}"); // TODO: Check this
tree.Parent = null;
tree.Children = TreeMappings[method].Forest.First().Children; // ??
//DictionaryHelper.AddOrAppend(TreeMappings, method, tree);
TreeMappings[method].Forest.Add(tree);
TreeIDs.Add(i, tree);
Model.Forest.Add(tree);
// UNIT TESTING: All contained methods at this point will have a parent.
// string.Join is being added as a method by a instruction, so when we try to patch it, it will have already a reference on the dictionary
// Here, we check if the method was already added by a instruction CALL
// Logic: If the method is already contained by the mapping dictionary
// then, we will exit adding a new that will have the same childs but a new ID
return false;
}
TreeIDs.Add(i, tree);
tree.FillMethodName($"{method.GetMethodSignature()}_{i}"); // TODO: Check this
foreach (var pair in instructions)
{
var opcode = pair.Key;
if (opcode != OpCodes.Call || opcode != OpCodes.Callvirt) continue;
var childMethod = (MethodBase)pair.Value;
var id = childMethod.GetID(out var _contains);
var subTree = new TreeModel(tree)
{
ID = id
};
if (_contains)
{
id = GetFreeID();
subTree.ID = id;
subTree.FillMethodName($"{childMethod.GetMethodSignature()}_{id}"); // TODO: Check this
subTree.Parent = TreeIDs[i];
subTree.Children = TreeMappings[childMethod].Forest.First().Children;
TreeIDs.Add(id, subTree);
continue;
}
TreeIDs.Add(id, subTree);
subTree.FillMethodName($"{childMethod.GetMethodSignature()}_{id}");
tree.Children.Add(subTree);
TreeMappings.Add(childMethod, new BenchmarkTreeModel());
TreeMappings[childMethod].Forest.Add(subTree);
}
TreeMappings.Add(method, new BenchmarkTreeModel());
TreeMappings[method].Forest.Add(tree);
Model.Forest.Add(tree);
return true;
//var treeModel = new TreeModel();
}
catch (Exception ex)
{
//Debug.LogException(new Exception(method.GetMethodSignature(), ex));
return false;
}
}, sw);
//return methods;
}
The GetMethodSignature is something like:
public static string GetMethodSignature(this MethodBase method)
{
if (method == null) return null;
return method.DeclaringType == null ? method.Name : $"{method.DeclaringType.FullName}.{method.Name}";
}
I think I'll replace it with the MethodBase.ToString instead (what do you think?)
Also, we have the BenchmarkCall class which allow us to take in care how many times the call was done and how many time it has spent at all:
[Serializable]
public class BenchmarkCall
{
public string Method { get; set; }
public double SpentMilliseconds { get; set; }
public long SpentTicks { get; set; }
public double MinSpentMs { get; set; } = double.MaxValue;
public double MaxSpentMs { get; set; } = double.MinValue;
public long MinSpentTicks { get; set; } = long.MaxValue;
public long MaxSpentTicks { get; set; } = long.MinValue;
public double AvgMs => SpentMilliseconds / TimesCalled;
public double AvgTicks => SpentTicks / (double)TimesCalled;
public BenchmarkCall()
{
}
public BenchmarkCall(MethodBase method)
{
Method = method.GetMethodSignature();
}
public override string ToString()
{
if (TimesCalled > 0)
return "BenchmarkCall{\n" +
$"Ticks[SpentTicks={SpentTicks},MinTicks={MinSpentTicks},MaxTicks={MaxSpentTicks},AvgTicks={AvgTicks:F2}]\n" +
$"Ms[SpentMs={SpentMilliseconds:F2},MinMs={MinSpentMs:F2},MaxMs={MaxSpentMs:F2},AvgMs={AvgMs:F2}]\n" +
"}";
return "BenchmarkCall{}";
}
}
}
So I think that my next movement will be to differentiate between X method being called from A or B (Xa or Xb) taking care of the full hierarchy (which I'm not sure how to do) instead of the parent method that calls it, maybe the code I wrote has some to do it with it, but I'm not sure (last night I was so tired, so I didn't code it taking care those facts), build up a list of method signatures with different IDs, and then fill up the tree, ID 1 is Xa and ID 2 is Xb (where I have problems also filling up the tree).
Also I'll need to use the Transpiler in order to alter all code instructions, so if a method has:
void method() {
X1();
X2();
}
We will need to add 2 methods (like prefix/postfix) to measure each instruction call:
void method() {
Start(1);
X1();
End(1);
Start(2);
X2();
End(2);
}
This will be a hard task, but I hope somebody could guide me with this out.