Unique values (dotnet#407)

* don't allow for duplicated arguments, fixes dotnet#404

* remove duplicated values, fixes dotnet#403

Author: adamsitnik

src/benchmarks/micro/corefx/System.Runtime/Perf.UInt16.cs

@@ -17,7 +17,7 @@ public class Perf_UInt16
         public static IEnumerable<object> Values => new object[]
         {
             ushort.MinValue,
-            0,
+            (ushort)12345, // same value used by other tests to compare the perf
             ushort.MaxValue
         };
 

src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs

@@ -25,6 +25,7 @@ public static IConfig Create(DirectoryInfo artifactsPath, ImmutableHashSet<strin
                 .With(JsonExporter.Full) // make sure we export to Json (for BenchView integration purpose)
                 .With(StatisticColumn.Median, StatisticColumn.Min, StatisticColumn.Max)
                 .With(TooManyTestCasesValidator.FailOnError)
+                .With(new UniqueArgumentsValidator()) // don't allow for duplicated arguments #404
                 .With(new MandatoryCategoryValidator(mandatoryCategories));
     }
 }

src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs

@@ -0,0 +1,42 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using BenchmarkDotNet.Validators;
+using System.Collections.Generic;
+using System.Linq;
+using BenchmarkDotNet.Running;
+
+namespace BenchmarkDotNet.Extensions
+{
+    public class UniqueArgumentsValidator : IValidator
+    {
+        public bool TreatsWarningsAsErrors => true;
+
+        public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
+            => validationParameters.Benchmarks
+                .Where(benchmark => benchmark.HasArguments || benchmark.HasParameters)
+                .GroupBy(benchmark => (benchmark.Descriptor.Type, benchmark.Descriptor.WorkloadMethod, benchmark.Job))
+                .Where(sameBenchmark =>
+                {
+                    int numberOfUniqueTestCases = sameBenchmark.Distinct(new BenchmarkArgumentsComparer()).Count();
+                    int numberOfTestCases = sameBenchmark.Count();
+
+                    return numberOfTestCases != numberOfUniqueTestCases;
+                })
+                .Select(duplicate => new ValidationError(true, $"Benchmark Arguments should be unique, {duplicate.Key.Type}.{duplicate.Key.WorkloadMethod} has duplicate arguments.", duplicate.First()));
+
+        private class BenchmarkArgumentsComparer : IEqualityComparer<BenchmarkCase>
+        {
+            public bool Equals(BenchmarkCase x, BenchmarkCase y)
+                => Enumerable.SequenceEqual(
+                    x.Parameters.Items.Select(argument => argument.Value), 
+                    y.Parameters.Items.Select(argument => argument.Value));
+
+            public int GetHashCode(BenchmarkCase obj)
+                => obj.Parameters.Items
+                    .Where(item => item.Value != null)
+                    .Aggregate(seed: 0, (hashCode, argument) => hashCode ^= argument.Value.GetHashCode());
+        }
+    }
+}

src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueArgumentsValidatorTests.cs

@@ -0,0 +1,91 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Validators;
+using Xunit;
+
+namespace Tests
+{
+    public class UniqueArgumentsValidatorTests
+    {
+        [Theory]
+        [InlineData(typeof(WithDuplicateValueTypeArguments), true)]
+        [InlineData(typeof(WithDuplicateReferenceTypeArguments), true)]
+        [InlineData(typeof(WithoutDuplicatedValueTypeArguments), false)]
+        [InlineData(typeof(WithoutDuplicatedReferenceTypeArguments), false)]
+        public void DuplicatedArgumentsAreDetected(Type typeWithBenchmarks, bool shouldReportError)
+        {
+            var benchmarksForType = BenchmarkConverter.TypeToBenchmarks(typeWithBenchmarks);
+            var validationParameters = new ValidationParameters(benchmarksForType.BenchmarksCases, benchmarksForType.Config);
+
+            var validationErrors = new UniqueArgumentsValidator().Validate(validationParameters);
+
+            if (shouldReportError)
+                Assert.NotEmpty(validationErrors);
+            else
+                Assert.Empty(validationErrors);
+        }
+
+        public class WithDuplicateValueTypeArguments
+        {
+            public static IEnumerable<object> Values => new object[]
+            {
+                ushort.MinValue, // 0
+                (ushort)0, // ushort.MinValue
+                ushort.MaxValue
+            };
+
+            [Benchmark]
+            [ArgumentsSource(nameof(Values))]
+            public string ToString(ushort value) => value.ToString();
+        }
+
+        public class WithDuplicateReferenceTypeArguments
+        {
+            public static IEnumerable<object> Values => new object[]
+            {
+                "",
+                string.Empty,
+                "something"
+            };
+
+            [Benchmark]
+            [ArgumentsSource(nameof(Values))]
+            public int GetHashCode(string value) => value.GetHashCode();
+        }
+
+        public class WithoutDuplicatedValueTypeArguments
+        {
+            public static IEnumerable<object> Values => new object[]
+            {
+                ushort.MinValue,
+                ushort.MaxValue
+            };
+
+            [Benchmark]
+            [ArgumentsSource(nameof(Values))]
+            public string ToString(ushort value) => value.ToString();
+        }
+
+        public class WithoutDuplicatedReferenceTypeArguments
+        {
+            public static IEnumerable<object> Values => new object[]
+            {
+                "",
+                "something"
+            };
+
+            [Benchmark]
+            [ArgumentsSource(nameof(Values))]
+            public int GetHashCode(string value) => value.GetHashCode();
+        }
+    }
+}