No Description
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

BenchmarkReportData.cs 9.1KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263
  1. using System;
  2. using Unity.Collections;
  3. using Unity.Collections.LowLevel.Unsafe;
  4. namespace Unity.PerformanceTesting.Benchmark
  5. {
  6. /// <summary>
  7. /// Specifies the statistic used for benchmark comparisons.
  8. /// </summary>
  9. public enum BenchmarkRankingStatistic
  10. {
  11. /// <summary>Compare the minimum time from a set of samples</summary>
  12. Min,
  13. /// <summary>Compare the maximum time from a set of samples</summary>
  14. Max,
  15. /// <summary>Compare the median time from a set of samples</summary>
  16. Median,
  17. /// <summary>Compare the average time from a set of samples</summary>
  18. Average,
  19. /// <summary>Compare the standard deviation of time from a set of samples</summary>
  20. StdDev,
  21. /// <summary>Compare the sum time of a set of samples</summary>
  22. Sum,
  23. }
  24. internal enum BenchmarkResultType
  25. {
  26. Ignored,
  27. Normal,
  28. NormalBaseline,
  29. External,
  30. ExternalBaseline,
  31. }
  32. internal enum BenchmarkRankingType
  33. {
  34. Ignored,
  35. Normal,
  36. Best,
  37. Worst,
  38. }
  39. internal struct BenchmarkResults
  40. {
  41. public const uint kFlagNoOptimization = 0x01;
  42. public const uint kFlagParallelJobs = 0x02;
  43. public const uint kFlagFootnotes = 0x04; // this must always be the last predefined flag bit
  44. public SampleUnit unit;
  45. public double min;
  46. public double max;
  47. public double median;
  48. public double average;
  49. public double standardDeviation;
  50. public double sum;
  51. public BenchmarkRankingType ranking;
  52. public BenchmarkRankingStatistic statistic;
  53. public double baselineRatio;
  54. public uint resultFlags;
  55. internal static readonly BenchmarkResults Ignored = new BenchmarkResults { ranking = BenchmarkRankingType.Ignored };
  56. internal BenchmarkResults(SampleGroup sampleGroup, BenchmarkRankingStatistic rankingStatistic, uint flags)
  57. {
  58. unit = sampleGroup.Unit;
  59. min = sampleGroup.Min;
  60. max = sampleGroup.Max;
  61. median = sampleGroup.Median;
  62. average = sampleGroup.Average;
  63. standardDeviation = sampleGroup.StandardDeviation;
  64. sum = sampleGroup.Sum;
  65. ranking = BenchmarkRankingType.Normal;
  66. statistic = rankingStatistic;
  67. baselineRatio = 0;
  68. resultFlags = flags;
  69. }
  70. public double Comparator
  71. {
  72. get
  73. {
  74. switch (statistic)
  75. {
  76. case BenchmarkRankingStatistic.Min: return min;
  77. case BenchmarkRankingStatistic.Max: return max;
  78. case BenchmarkRankingStatistic.Median: return median;
  79. case BenchmarkRankingStatistic.Average: return average;
  80. case BenchmarkRankingStatistic.StdDev: return standardDeviation;
  81. case BenchmarkRankingStatistic.Sum: return sum;
  82. }
  83. return median;
  84. }
  85. }
  86. public string UnitSuffix
  87. {
  88. get
  89. {
  90. switch (unit)
  91. {
  92. case SampleUnit.Nanosecond: return "ns";
  93. case SampleUnit.Microsecond: return "µs";
  94. case SampleUnit.Millisecond: return "ms";
  95. case SampleUnit.Second: return "s";
  96. case SampleUnit.Byte: return "b";
  97. case SampleUnit.Kilobyte: return "kb";
  98. case SampleUnit.Megabyte: return "mb";
  99. case SampleUnit.Gigabyte: return "gb";
  100. case SampleUnit.Undefined:
  101. break;
  102. }
  103. return "";
  104. }
  105. }
  106. }
  107. internal struct BenchmarkReportComparison : IDisposable
  108. {
  109. public UnsafeList<BenchmarkResults> results;
  110. public FixedString512Bytes comparisonName;
  111. public uint footnoteFlags;
  112. public BenchmarkReportComparison(string name)
  113. {
  114. results = new UnsafeList<BenchmarkResults>(1, Allocator.Persistent);
  115. comparisonName = name;
  116. footnoteFlags = 0;
  117. }
  118. public void Dispose()
  119. {
  120. if (results.IsCreated)
  121. results.Dispose();
  122. }
  123. public void RankResults(BenchmarkResultType[] resultTypes)
  124. {
  125. double min = double.MaxValue;
  126. double max = double.MinValue;
  127. int baselineJ = -1;
  128. int firstJ = -1;
  129. for (int j = 0; j < results.Length; j++)
  130. {
  131. if (results[j].ranking == BenchmarkRankingType.Ignored)
  132. continue;
  133. if (firstJ == -1)
  134. firstJ = j;
  135. double result = results[j].Comparator;
  136. if (result < min)
  137. min = result;
  138. if (result > max)
  139. max = result;
  140. if (resultTypes[j] == BenchmarkResultType.ExternalBaseline || resultTypes[j] == BenchmarkResultType.NormalBaseline)
  141. {
  142. if (baselineJ == -1)
  143. baselineJ = j;
  144. else
  145. throw new Exception("[INTERNAL ERROR] More than one baseline found - this should have been caught during initialization");
  146. }
  147. }
  148. bool same = true;
  149. for (int prevJ = firstJ, j = firstJ + 1; j < results.Length; j++)
  150. {
  151. if (results[j].ranking == BenchmarkRankingType.Ignored)
  152. continue;
  153. if (results[prevJ].Comparator != results[j].Comparator)
  154. same = false;
  155. prevJ = j;
  156. }
  157. if (!same)
  158. {
  159. for (int j = 0; j < results.Length; j++)
  160. {
  161. if (results[j].ranking == BenchmarkRankingType.Ignored)
  162. continue;
  163. if (results[j].Comparator == min)
  164. results.ElementAt(j).ranking = BenchmarkRankingType.Best;
  165. else if (results[j].Comparator == max)
  166. results.ElementAt(j).ranking = BenchmarkRankingType.Worst;
  167. }
  168. }
  169. if (baselineJ == -1)
  170. throw new Exception("[INTERNAL ERROR] No baseline found - this should have been caught during initialization");
  171. for (int j = 0; j < results.Length; j++)
  172. {
  173. if (results[j].ranking == BenchmarkRankingType.Ignored)
  174. continue;
  175. if (results[j].Comparator != 0)
  176. results.ElementAt(j).baselineRatio = results[baselineJ].Comparator / results[j].Comparator;
  177. }
  178. }
  179. }
  180. internal struct BenchmarkReportGroup : IDisposable
  181. {
  182. public UnsafeList<BenchmarkReportComparison> comparisons;
  183. public FixedString512Bytes groupName;
  184. public UnsafeList<FixedString64Bytes> variantNames;
  185. public UnsafeList<BenchmarkResultType> resultTypes;
  186. public int resultDecimalPlaces;
  187. public UnsafeHashMap<uint, NativeText> customFootnotes;
  188. public BenchmarkReportGroup(string name, string[] variantNameArray, BenchmarkResultType[] resultTypeArray, int resultDecimalPlaces)
  189. {
  190. comparisons = new UnsafeList<BenchmarkReportComparison>(1, Allocator.Persistent);
  191. groupName = name;
  192. variantNames = new UnsafeList<FixedString64Bytes>(variantNameArray.Length, Allocator.Persistent);
  193. resultTypes = new UnsafeList<BenchmarkResultType>(resultTypeArray.Length, Allocator.Persistent);
  194. this.resultDecimalPlaces = resultDecimalPlaces;
  195. foreach (var title in variantNameArray)
  196. variantNames.Add(title);
  197. foreach (var resultType in resultTypeArray)
  198. resultTypes.Add(resultType);
  199. customFootnotes = new UnsafeHashMap<uint, NativeText>(30, Allocator.Persistent);
  200. }
  201. public void Dispose()
  202. {
  203. if (comparisons.IsCreated)
  204. {
  205. for (int i = 0; i < comparisons.Length; i++)
  206. comparisons[i].Dispose();
  207. comparisons.Dispose();
  208. }
  209. if (variantNames.IsCreated)
  210. variantNames.Dispose();
  211. if (customFootnotes.IsCreated)
  212. {
  213. foreach (var pair in customFootnotes)
  214. pair.Value.Dispose();
  215. customFootnotes.Dispose();
  216. }
  217. }
  218. }
  219. internal struct BenchmarkReports : IDisposable
  220. {
  221. public UnsafeList<BenchmarkReportGroup> groups;
  222. public FixedString512Bytes reportName;
  223. public BenchmarkReports(string name)
  224. {
  225. groups = new UnsafeList<BenchmarkReportGroup>(1, Allocator.Persistent);
  226. reportName = name;
  227. }
  228. public void Dispose()
  229. {
  230. if (groups.IsCreated)
  231. {
  232. for (int i = 0; i < groups.Length; i++)
  233. groups[i].Dispose();
  234. groups.Dispose();
  235. }
  236. }
  237. }
  238. }