Fix Confidence Filter at Rule Level (#672)

* Update .gitignore

* Fix confidence filter at rule level.

* Update Changelog.md

* Add tests for rule level confidence filtering

* Clarify Doc comment on WithConfidenceFilter method
Update options tests to account for unspecified behavior (its the 0 value for the enum flag so always passes the check).

Author: gfs

.gitignore

@@ -17,3 +17,6 @@ DevSkim-DotNet/Microsoft.DevSkim.VisualStudio/devskim-server-*.txt
 
 # Legacy Files
 DevSkim-VSCode-Plugin/server/
+
+# Mac OS Metadata
+**/.DS_Store

Changelog.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.51] - 2024-12-09
+## Fix
+Fix confidence filtering at rule level.
+
 ## [1.0.50] - 2024-12-05
 ## Fix
 Fixes #664 handling of options from IgnoreRuleMap when using OptionsJson

DevSkim-DotNet/Microsoft.DevSkim.Tests/OptionsTests.cs

@@ -362,4 +362,170 @@ public void TestParsingJsonOptions()
         // This should be 0, because the globs exclude js files
         Assert.AreEqual(0, analyzerWithSerialized.Run());
     }
+    
+    DevSkimRule highConfidenceRule = new DevSkimRule()
+    {
+        Name = "Weak/Broken Hash Algorithm",
+        Id = "HighConfidence",
+        Description = "Confidence Filter Tests",
+        Tags = new List<string>() { "Tests.ConfidenceFilter" },
+        Severity = Severity.Critical,
+        Confidence = Confidence.High,
+        Patterns = new[]
+        {
+            new SearchPattern()
+            {
+                Pattern = "Hello",
+                PatternType = PatternType.Regex,
+                Scopes = new[]
+                {
+                    PatternScope.All
+                }
+            }
+        }
+    };
+    
+    DevSkimRule mediumConfidenceRule = new DevSkimRule()
+    {
+        Name = "Weak/Broken Hash Algorithm",
+        Id = "MediumConfidence",
+        Description = "Confidence Filter Tests",
+        Tags = new List<string>() { "Tests.ConfidenceFilter" },
+        Severity = Severity.Critical,
+        Confidence = Confidence.Medium,
+        Patterns = new[]
+        {
+            new SearchPattern()
+            {
+                Pattern = "Hello",
+                PatternType = PatternType.Regex,
+                Scopes = new[]
+                {
+                    PatternScope.All
+                }
+            }
+        }
+    };
+    
+    DevSkimRule lowConfidenceRule = new DevSkimRule()
+    {
+        Name = "Weak/Broken Hash Algorithm",
+        Id = "LowConfidence",
+        Description = "Confidence Filter Tests",
+        Tags = new List<string>() { "Tests.ConfidenceFilter" },
+        Severity = Severity.Critical,
+        Confidence = Confidence.Low,
+        Patterns = new[]
+        {
+            new SearchPattern()
+            {
+                Pattern = "Hello",
+                PatternType = PatternType.Regex,
+                Scopes = new[]
+                {
+                    PatternScope.All
+                }
+            }
+        }
+    };
+    
+    DevSkimRule unspecifiedConfidenceRule = new DevSkimRule()
+    {
+        Name = "Weak/Broken Hash Algorithm",
+        Id = "UnspecifiedConfidence",
+        Description = "Confidence Filter Tests",
+        Tags = new List<string>() { "Tests.ConfidenceFilter" },
+        Severity = Severity.Critical,
+        Confidence = Confidence.Unspecified,
+        Patterns = new[]
+        {
+            new SearchPattern()
+            {
+                Pattern = "Hello",
+                PatternType = PatternType.Regex,
+                Scopes = new[]
+                {
+                    PatternScope.All
+                }
+            }
+        }
+    };
+    
+    [TestMethod]
+    public void TestConfidenceFiltering()
+    {
+        
+        var ruleSet = new DevSkimRuleSet();
+        ruleSet.AddRule(highConfidenceRule);
+        ruleSet.AddRule(mediumConfidenceRule);
+        ruleSet.AddRule(lowConfidenceRule);
+        ruleSet.AddRule(unspecifiedConfidenceRule);
+        Assert.AreEqual(4, ruleSet.Count());
+        // Unspecified always passes the confidence filter
+        Assert.AreEqual(2, 
+            ruleSet.WithConfidenceFilter(Confidence.High)
+                .Count());
+        Assert.AreEqual(2, 
+            ruleSet.WithConfidenceFilter(Confidence.Medium)
+                .Count());
+        Assert.AreEqual(2, 
+            ruleSet.WithConfidenceFilter(Confidence.Low)
+                .Count());
+        Assert.AreEqual(1, 
+            ruleSet.WithConfidenceFilter(Confidence.Unspecified)
+                .Count());
+    }
+
+    [TestMethod]
+    public void TestFullFlowWithConfidenceFiltering()
+    {
+        var rulesContent = new List<DevSkimRule>()
+        {
+            unspecifiedConfidenceRule,
+            lowConfidenceRule,
+            mediumConfidenceRule,
+            highConfidenceRule
+        };
+        var testContent = "Hello";
+        var rulesPath = PathHelper.GetRandomTempFile("json");
+        var csharpTestPath = PathHelper.GetRandomTempFile("cs");
+        {
+            using var csharpStream = File.Create(csharpTestPath);
+            JsonSerializer.Serialize(csharpStream, testContent);
+            File.WriteAllText(rulesPath, JsonSerializer.Serialize(rulesContent));
+        }
+
+        var confidenceOptions = new AnalyzeCommandOptions()
+        {
+            Path = csharpTestPath,
+            Rules = new[] { rulesPath },
+            Confidences = new [] { Confidence.High, Confidence.Medium, Confidence.Low, Confidence.Unspecified },
+            ExitCodeIsNumIssues = true
+        };
+
+        var analyzerWithSerialized = new AnalyzeCommand(confidenceOptions);
+        Assert.AreEqual(4, analyzerWithSerialized.Run());
+
+        confidenceOptions.Confidences = new[] { Confidence.High };
+
+        // Unspecified confidence rules are not filtered out because confidence may have been
+        //  (and should be set) at pattern level
+        analyzerWithSerialized = new AnalyzeCommand(confidenceOptions);
+        Assert.AreEqual(2, analyzerWithSerialized.Run());
+        
+        confidenceOptions.Confidences = new[] { Confidence.Medium };
+        
+        analyzerWithSerialized = new AnalyzeCommand(confidenceOptions);
+        Assert.AreEqual(2, analyzerWithSerialized.Run());
+        
+        confidenceOptions.Confidences = new[] { Confidence.Low };
+        
+        analyzerWithSerialized = new AnalyzeCommand(confidenceOptions);
+        Assert.AreEqual(2, analyzerWithSerialized.Run());
+
+        confidenceOptions.Confidences = new[] { Confidence.Unspecified };
+
+        analyzerWithSerialized = new AnalyzeCommand(confidenceOptions);
+        Assert.AreEqual(1, analyzerWithSerialized.Run());
+    }
 }

DevSkim-DotNet/Microsoft.DevSkim/DevSkimRuleProcessor.cs

@@ -17,6 +17,8 @@ public class DevSkimRuleProcessor
 
         public DevSkimRuleProcessor(DevSkimRuleSet ruleSet, DevSkimRuleProcessorOptions processorOptions)
         {
+            // Application Inspector Processor filters *patterns* based on confidence but not rules
+            ruleSet = ruleSet.WithConfidenceFilter(processorOptions.ConfidenceFilter);
             _aiProcessor = new RuleProcessor(ruleSet, processorOptions);
             _processorOptions = processorOptions;
         }

DevSkim-DotNet/Microsoft.DevSkim/DevSkimRuleSet.cs

@@ -40,6 +40,18 @@ public static DevSkimRuleSet GetDefaultRuleSet()
             return ruleSet;
         }
 
+        /// <summary>
+        /// Return a new RuleSet containing only rules that have one of the flags of the specified confidence enum, or Unspecified
+        /// </summary>
+        /// <param name="filter">The Enum with flags set for which Confidence rules to use</param>
+        /// <returns>A new DevSkimRuleSet with only rules that have the specified confidence set at the Rule level</returns>
+        public DevSkimRuleSet WithConfidenceFilter(Confidence filter)
+        {
+            DevSkimRuleSet newSet = new DevSkimRuleSet();
+            newSet.AddRange(this.Where(x => filter.HasFlag(x.Confidence)));
+            return newSet;
+        }
+
         /// <summary>
         /// Returns a new <see cref="DevSkimRuleSet"/> with only rules that have an ID matching one of the ids provided in <paramref name="ruleIds"/>
         /// </summary>