microsoft · michaelcfanning · Dec 5, 2018 · Dec 5, 2018
diff --git a/src/ReleaseHistory.md b/src/ReleaseHistory.md
@@ -53,7 +53,7 @@
 * Add SuppressedExternally to SuppressionStates enum
 
 ## **v1.5.26** [Driver](https://www.nuget.org/packages/Sarif.Driver/1.5.26) | [SDK](https://www.nuget.org/packages/Sarif.Sdk/1.5.26)  
-* Breaking change on SarifLogger to explicitly specify hash computation for all files  
+* API BREAKING change on SarifLogger to explicitly specify hash computation for all files  
 * SarifLogger now automatically persists file data for all URIs through format  
 * Add run.stableId, a consistent run-over-run log identifier  
 * Add annotatedCodeLocation.callee and annotatedCodeLocation.calleeKey for annotation call sites  
@@ -64,7 +64,7 @@
 * Ship checked in CommandLine.dll in order to allow this 'beta' NuGet component to ship in Driver non-beta release
 
 ## **v1.5.28** [Driver](https://www.nuget.org/packages/Sarif.Driver/1.5.28) | [SDK](https://www.nuget.org/packages/Sarif.Sdk/1.5.28)
-* Breaking change: rename PropertyBagDictionary to PropertiesDictionary
+* API BREAKING change: rename PropertyBagDictionary to PropertiesDictionary
 * Add 'functionReturn' to annotatedCodeLocation.kind
 * Remove 'source', 'sink' and 'sanitizer' from annotatedCodeLocation.kind
 * Add 'taint' enum to annotatedCodeLocation with values 'source', 'sink' and 'sanitizer'
@@ -82,7 +82,7 @@
 
 ## **v1.5.31** [Driver](https://www.nuget.org/packages/Sarif.Driver/1.5.31) | [SDK](https://www.nuget.org/packages/Sarif.Sdk/1.5.31)
 
-* BREAKING CHANGE: RuleUtilities.BuildResult no longer automatically prepends the target file path to the list of FormattedRuleMessage.Arguments array in the Result object being built.
+* API BREAKING CHANGE: RuleUtilities.BuildResult no longer automatically prepends the target file path to the list of FormattedRuleMessage.Arguments array in the Result object being built.
 
 ## **v1.5.32** [Driver](https://www.nuget.org/packages/Sarif.Driver/1.5.32) | [SDK](https://www.nuget.org/packages/Sarif.Sdk/1.5.32)
 
@@ -193,33 +193,34 @@
 * Provide SARIF v1.0 object model and v1 <-> v2 transformation API
 
 ## **v2.0.0-csd.1.0.1** [Sdk](https://www.nuget.org/packages/Sarif.Sdk/2.0.0-csd.1.0.1) | [Driver](https://www.nuget.org/packages/Sarif.Driver/2.0.0-csd.1.0.1) | [Converters](https://www.nuget.org/packages/Sarif.Converters/2.0.0-csd.1.0.1)) | [Multitool](https://www.nuget.org/packages/Sarif.Multitool/2.0.0-csd.1.0.1))
-* BREAKING CHANGE: Fix weakly typed CreateNotification calls and make API more strongly typed
-* BREAKING CHANGE: Rename OptionallyEmittedData.ContextCodeSnippets to ContextRegionSnippets
-* BREAKING CHANGE: Eliminate result.ruleMessageId (in favor of result.message.messageId)
+* API BREAKING CHANGE: Fix weakly typed CreateNotification calls and make API more strongly typed
+* API BREAKING CHANGE: Rename OptionallyEmittedData.ContextCodeSnippets to ContextRegionSnippets
+* API BREAKING CHANGE: Eliminate result.ruleMessageId (in favor of result.message.messageId)
 
 ## **v2.0.0-csd.1.0.2** [Sdk](https://www.nuget.org/packages/Sarif.Sdk/2.0.0-csd.1.0.2) | [Driver](https://www.nuget.org/packages/Sarif.Driver/2.0.0-csd.1.0.2) | [Converters](https://www.nuget.org/packages/Sarif.Converters/2.0.0-csd.1.0.2)) | [Multitool](https://www.nuget.org/packages/Sarif.Multitool/2.0.0-csd.1.0.2))
 * Bugfix to result matching algorithm where empty or null previous log sets caused a NullReferenceException.
 * Bugfix to result matching algorithm where we were incorrectly detecting duplicate data across files, and changed a "NotImplementedException" to the correct "InvalidOperationException".
 
 ## **v2.0.0-csd.2.beta.2018.10.10** [Sdk](https://www.nuget.org/packages/Sarif.Sdk/2.0.0-csd.2.beta.2018.10.10) | [Driver](https://www.nuget.org/packages/Sarif.Driver/2.0.0-csd.2.beta.2018.10.10) | [Converters](https://www.nuget.org/packages/Sarif.Converters/2.0.0-csd.2.beta.2018.10.10)) | [Multitool](https://www.nuget.org/packages/Sarif.Multitool/2.0.0-csd.2.beta.2018.10.10))
-* BREAKING: invocation.workingDirectory is now a FileLocation object (and not a URI expressed as a string)
-* Add run.externalFiles object to schema. Sync generally to OASIS TC schema.
-* Add --sarif-version command to driver (to transform SARIF output to v1 format)
-* Fix bug that persisted tokens to redact as file references.
-* BREAKING: originalUriBaseIds is now a dictionary of file locations, not strings.
-* BREAKING: Suffix invocation.startTime, invocation.endTime, file.lastModifiedTime and notification.time with Utc (startTimeUtc, endTimeUtc, etc.).
-* BREAKING: threadflowLocation.timestamp renamed to 'executionTimeUtc'.
-* BREAKING: versionControlDetails.timestamp renamed to 'asOfTimeUtc'.
-* BREAKING: versionControlDetails.uri renamed to 'repositoryUri'.
-* BREAKING: versionControlDetails.tag renamed to 'revisionTag'
-* BREAKING: exception.message type converted from string to message object.
-* BREAKING: file.hashes is now a string/string dictionary, not an array of 'hash' objects (the type for which is deleted)
-* BREAKING: run.instanceGuid, run.correlationGuid, run.logicalId, run.description combined into new run.id 'runAutomationDetails' object instance.
-* BREAKING: run.automationLogicalId subsumed by run.aggregateIds, an array of 'runAutomationDetails' objects.
-* BREAKING: Remove threadFlowLocation.step
-* Add result.occurrenceCount (denotes # of occurrences of an identical results within an analysisRun)
+* FEATURE:Add --sarif-version command to driver (to transform SARIF output to v1 format)
+* BUGFIX: Drop erroneous persistence of redaction tokens as files objects.
+* API NON-BREAKING: Add result.occurrenceCount (denotes # of occurrences of an identical results within an analysisRun)
+* API NON-BREAKING: Add run.externalFiles object to schema. Sync generally to OASIS TC schema.
+* API BREAKING: originalUriBaseIds is now a dictionary of file locations, not strings.
+* API BREAKING: Suffix invocation.startTime, invocation.endTime, file.lastModifiedTime and notification.time with Utc (startTimeUtc, endTimeUtc, etc.).
+* API BREAKING: threadflowLocation.timestamp renamed to 'executionTimeUtc'.
+* API BREAKING: versionControlDetails.timestamp renamed to 'asOfTimeUtc'.
+* API BREAKING: versionControlDetails.uri renamed to 'repositoryUri'.
+* API BREAKING: versionControlDetails.tag renamed to 'revisionTag'
+* API BREAKING: exception.message type converted from string to message object.
+* API BREAKING: file.hashes is now a string/string dictionary, not an array of 'hash' objects (the type for which is deleted)
+* API BREAKING: run.instanceGuid, run.correlationGuid, run.logicalId, run.description combined into new run.id 'runAutomationDetails' object instance.
+* API BREAKING: run.automationLogicalId subsumed by run.aggregateIds, an array of 'runAutomationDetails' objects.
+* API BREAKING: Remove threadFlowLocation.step
+* API BREAKING: invocation.workingDirectory is now a FileLocation object (and not a URI expressed as a string)
 
 ## **v2.0.0-csd.2.beta.2018.11.28** [Sdk](https://www.nuget.org/packages/Sarif.Sdk/2.0.0-csd.2.beta.2018.11.28) | [Driver](https://www.nuget.org/packages/Sarif.Driver/2.0.0-csd.2.beta.2018.11.28) | [Converters](https://www.nuget.org/packages/Sarif.Converters/2.0.0-csd.2.beta.2018.11.28)) | [Multitool](https://www.nuget.org/packages/Sarif.Multitool/2.0.0-csd.2.beta.2018.11.28))
-* Result matching improvements
-* Fortify FPR converter improvements
-* BREAKING: remove run.architecture /~https://github.com/oasis-tcs/sarif-spec/issues/262
+* BUGFIX: Result matching improvements in properties persistence
+* FEATURE: Fortify FPR converter improvements
+* API NON-BREAKING: Add run.newlineSequences to schema. /~https://github.com/oasis-tcs/sarif-spec/issues/169
+* API BREAKING: remove run.architecture /~https://github.com/oasis-tcs/sarif-spec/issues/262
diff --git a/src/Sarif/Autogenerated/Run.cs b/src/Sarif/Autogenerated/Run.cs
@@ -128,6 +128,12 @@ public SarifNodeKind SarifNodeKind
         [DataMember(Name = "defaultFileEncoding", IsRequired = false, EmitDefaultValue = false)]
         public string DefaultFileEncoding { get; set; }
 
+        /// <summary>
+        /// An ordered list of character sequences that were treated as line breaks when computing region information for the run.
+        /// </summary>
+        [DataMember(Name = "newlineSequences", IsRequired = false, EmitDefaultValue = false)]
+        public IList<string> NewlineSequences { get; set; }
+
         /// <summary>
         /// Specifies the unit in which the tool measures columns.
         /// </summary>
@@ -198,15 +204,18 @@ public Run()
         /// <param name="defaultFileEncoding">
         /// An initialization value for the <see cref="P: DefaultFileEncoding" /> property.
         /// </param>
+        /// <param name="newlineSequences">
+        /// An initialization value for the <see cref="P: NewlineSequences" /> property.
+        /// </param>
         /// <param name="columnKind">
         /// An initialization value for the <see cref="P: ColumnKind" /> property.
         /// </param>
         /// <param name="properties">
         /// An initialization value for the <see cref="P: Properties" /> property.
         /// </param>
-        public Run(Tool tool, IEnumerable<Invocation> invocations, Conversion conversion, IEnumerable<VersionControlDetails> versionControlProvenance, IDictionary<string, FileLocation> originalUriBaseIds, IDictionary<string, FileData> files, IDictionary<string, LogicalLocation> logicalLocations, IDictionary<string, Graph> graphs, IEnumerable<Result> results, Resources resources, RunAutomationDetails id, IEnumerable<RunAutomationDetails> aggregateIds, string baselineInstanceGuid, string richMessageMimeType, string redactionToken, string defaultFileEncoding, ColumnKind columnKind, IDictionary<string, SerializedPropertyInfo> properties)
+        public Run(Tool tool, IEnumerable<Invocation> invocations, Conversion conversion, IEnumerable<VersionControlDetails> versionControlProvenance, IDictionary<string, FileLocation> originalUriBaseIds, IDictionary<string, FileData> files, IDictionary<string, LogicalLocation> logicalLocations, IDictionary<string, Graph> graphs, IEnumerable<Result> results, Resources resources, RunAutomationDetails id, IEnumerable<RunAutomationDetails> aggregateIds, string baselineInstanceGuid, string richMessageMimeType, string redactionToken, string defaultFileEncoding, IEnumerable<string> newlineSequences, ColumnKind columnKind, IDictionary<string, SerializedPropertyInfo> properties)
         {
-            Init(tool, invocations, conversion, versionControlProvenance, originalUriBaseIds, files, logicalLocations, graphs, results, resources, id, aggregateIds, baselineInstanceGuid, richMessageMimeType, redactionToken, defaultFileEncoding, columnKind, properties);
+            Init(tool, invocations, conversion, versionControlProvenance, originalUriBaseIds, files, logicalLocations, graphs, results, resources, id, aggregateIds, baselineInstanceGuid, richMessageMimeType, redactionToken, defaultFileEncoding, newlineSequences, columnKind, properties);
         }
 
         /// <summary>
@@ -225,7 +234,7 @@ public Run(Run other)
                 throw new ArgumentNullException(nameof(other));
             }
 
-            Init(other.Tool, other.Invocations, other.Conversion, other.VersionControlProvenance, other.OriginalUriBaseIds, other.Files, other.LogicalLocations, other.Graphs, other.Results, other.Resources, other.Id, other.AggregateIds, other.BaselineInstanceGuid, other.RichMessageMimeType, other.RedactionToken, other.DefaultFileEncoding, other.ColumnKind, other.Properties);
+            Init(other.Tool, other.Invocations, other.Conversion, other.VersionControlProvenance, other.OriginalUriBaseIds, other.Files, other.LogicalLocations, other.Graphs, other.Results, other.Resources, other.Id, other.AggregateIds, other.BaselineInstanceGuid, other.RichMessageMimeType, other.RedactionToken, other.DefaultFileEncoding, other.NewlineSequences, other.ColumnKind, other.Properties);
         }
 
         ISarifNode ISarifNode.DeepClone()
@@ -246,7 +255,7 @@ private ISarifNode DeepCloneCore()
             return new Run(this);
         }
 
-        private void Init(Tool tool, IEnumerable<Invocation> invocations, Conversion conversion, IEnumerable<VersionControlDetails> versionControlProvenance, IDictionary<string, FileLocation> originalUriBaseIds, IDictionary<string, FileData> files, IDictionary<string, LogicalLocation> logicalLocations, IDictionary<string, Graph> graphs, IEnumerable<Result> results, Resources resources, RunAutomationDetails id, IEnumerable<RunAutomationDetails> aggregateIds, string baselineInstanceGuid, string richMessageMimeType, string redactionToken, string defaultFileEncoding, ColumnKind columnKind, IDictionary<string, SerializedPropertyInfo> properties)
+        private void Init(Tool tool, IEnumerable<Invocation> invocations, Conversion conversion, IEnumerable<VersionControlDetails> versionControlProvenance, IDictionary<string, FileLocation> originalUriBaseIds, IDictionary<string, FileData> files, IDictionary<string, LogicalLocation> logicalLocations, IDictionary<string, Graph> graphs, IEnumerable<Result> results, Resources resources, RunAutomationDetails id, IEnumerable<RunAutomationDetails> aggregateIds, string baselineInstanceGuid, string richMessageMimeType, string redactionToken, string defaultFileEncoding, IEnumerable<string> newlineSequences, ColumnKind columnKind, IDictionary<string, SerializedPropertyInfo> properties)
         {
             if (tool != null)
             {
@@ -380,6 +389,17 @@ private void Init(Tool tool, IEnumerable<Invocation> invocations, Conversion con
             RichMessageMimeType = richMessageMimeType;
             RedactionToken = redactionToken;
             DefaultFileEncoding = defaultFileEncoding;
+            if (newlineSequences != null)
+            {
+                var destination_4 = new List<string>();
+                foreach (var value_8 in newlineSequences)
+                {
+                    destination_4.Add(value_8);
+                }
+
+                NewlineSequences = destination_4;
+            }
+
             ColumnKind = columnKind;
             if (properties != null)
             {

diff --git a/src/Sarif/Autogenerated/RunEqualityComparer.cs b/src/Sarif/Autogenerated/RunEqualityComparer.cs
@@ -240,6 +240,27 @@ public bool Equals(Run left, Run right)
                 return false;
             }
 
+            if (!object.ReferenceEquals(left.NewlineSequences, right.NewlineSequences))
+            {
+                if (left.NewlineSequences == null || right.NewlineSequences == null)
+                {
+                    return false;
+                }
+
+                if (left.NewlineSequences.Count != right.NewlineSequences.Count)
+                {
+                    return false;
+                }
+
+                for (int index_4 = 0; index_4 < left.NewlineSequences.Count; ++index_4)
+                {
+                    if (left.NewlineSequences[index_4] != right.NewlineSequences[index_4])
+                    {
+                        return false;
+                    }
+                }
+            }
+
             if (left.ColumnKind != right.ColumnKind)
             {
                 return false;
@@ -432,17 +453,29 @@ public int GetHashCode(Run obj)
                     result = (result * 31) + obj.DefaultFileEncoding.GetHashCode();
                 }
 
+                if (obj.NewlineSequences != null)
+                {
+                    foreach (var value_18 in obj.NewlineSequences)
+                    {
+                        result = result * 31;
+                        if (value_18 != null)
+                        {
+                            result = (result * 31) + value_18.GetHashCode();
+                        }
+                    }
+                }
+
                 result = (result * 31) + obj.ColumnKind.GetHashCode();
                 if (obj.Properties != null)
                 {
                     // Use xor for dictionaries to be order-independent.
                     int xor_4 = 0;
-                    foreach (var value_18 in obj.Properties)
+                    foreach (var value_19 in obj.Properties)
                     {
-                        xor_4 ^= value_18.Key.GetHashCode();
-                        if (value_18.Value != null)
+                        xor_4 ^= value_19.Key.GetHashCode();
+                        if (value_19.Value != null)
                         {
-                            xor_4 ^= value_18.Value.GetHashCode();
+                            xor_4 ^= value_19.Value.GetHashCode();
                         }
                     }
 

diff --git a/src/Sarif/Core/Run.cs b/src/Sarif/Core/Run.cs
@@ -20,5 +20,7 @@ public partial class Run
         public bool ShouldSerializeInvocations() { return this.Invocations != null && this.Invocations.Any((e) => e != null && !e.ValueEquals(EmptyInvocation)); }
 
         public bool ShouldSerializeLogicalLocations() { return this.LogicalLocations != null && this.LogicalLocations.Values.Any(); }
+
+        public bool ShouldSerializeNewlineSequences() { return this.NewlineSequences != null && this.NewlineSequences.Any((s) => s != null); }
     }
 }
diff --git a/src/Sarif/Schemata/sarif-schema.json b/src/Sarif/Schemata/sarif-schema.json
@@ -1571,6 +1571,17 @@
           "type": "string"
         },
 
+        "newlineSequences": {
+          "description": "An ordered list of character sequences that were treated as line breaks when computing region information for the run.",
+          "type": "array",
+          "minItems": 1,
+          "uniqueItems": true,
+          "default": [ "\r\n", "\n" ],
+          "items": {
+            "type": "string"
+          }
+        },
+
         "columnKind": {
           "description": "Specifies the unit in which the tool measures columns.",
           "enum": [ "utf16CodeUnits", "unicodeCodePoints" ]

diff --git a/src/Sarif/Writers/PrereleaseCompatibilityTransformer.cs b/src/Sarif/Writers/PrereleaseCompatibilityTransformer.cs
@@ -59,7 +59,11 @@ public static string UpdateToCurrentVersion(string prereleaseSarifLog, bool forc
 
         private static bool ApplyChangesFromTC25ThroughTC28(JObject sarifLog)
         {
-            bool modifiedLog = UpdateSarifLogVersion(sarifLog); 
+            bool modifiedLog = UpdateSarifLogVersion(sarifLog);
+
+            // For completness, this update added run.newlineSequences to the schema
+            // This is a non-breaking (additive) change, so there is no work to do.
+            ///~https://github.com/oasis-tcs/sarif-spec/issues/169
 
             var runs = (JArray)sarifLog["runs"];