Skip to content

Commit

Permalink
Merge pull request #8109 from SimonCropp/use-utf8-strings
Browse files Browse the repository at this point in the history 
use utf8 strings
  • Loading branch information
@MiYanni
MiYanni authored Jun 13, 2024
2 parents 5e4bdc5 + d13e2ba commit 9bcea72
Show file tree
Hide file tree
Showing 4 changed files with 44 additions and 44 deletions.
4 changes: 2 additions & 2 deletions test/Microsoft.TemplateEngine.Core.UnitTests/ChunkStreamReadTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,11 +130,11 @@ public void VerifyConsumeWholeLine()
return 0;
},
true,
Encoding.UTF8.GetBytes("There"));
"There"u8.ToArray());

EngineConfig cfg = new EngineConfig(_engineEnvironmentSettings.Host.Logger, new VariableCollection());
IProcessor processor = Processor.Create(cfg, o.Provider);
byte[] data = Encoding.UTF8.GetBytes("Hello \r\n There \r\n You");
byte[] data = "Hello \r\n There \r\n You"u8.ToArray();
Stream input = new ChunkMemoryStream(data, 1);
Stream output = new ChunkMemoryStream(1);
bool changed = processor.Run(input, output, 5);
Expand Down
40 changes: 20 additions & 20 deletions test/Microsoft.TemplateEngine.Core.UnitTests/CommonOperationsTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ public void VerifyTrimWhitespaceForward()
return 0;
},
true,
Encoding.UTF8.GetBytes("Hello"));
"Hello"u8.ToArray());

EngineConfig cfg = new EngineConfig(_logger, new VariableCollection());
IProcessor processor = Processor.Create(cfg, o.Provider);
byte[] data = Encoding.UTF8.GetBytes("Hello \r\n There");
byte[] data = "Hello \r\n There"u8.ToArray();
Stream d = new MemoryStream(data);
MemoryStream result = new MemoryStream();
bool modified = processor.Run(d, result);
Expand All @@ -56,11 +56,11 @@ public void VerifyTrimWhitespaceBackward()
return 0;
},
true,
Encoding.UTF8.GetBytes("There"));
"There"u8.ToArray());

EngineConfig cfg = new EngineConfig(_logger, new VariableCollection());
IProcessor processor = Processor.Create(cfg, o.Provider);
byte[] data = Encoding.UTF8.GetBytes("Hello \r\n There");
byte[] data = "Hello \r\n There"u8.ToArray();
Stream d = new MemoryStream(data);
MemoryStream result = new MemoryStream();
bool modified = processor.Run(d, result);
Expand All @@ -81,11 +81,11 @@ public void VerifyTrimWhitespaceBothDirections()
return 0;
},
true,
Encoding.UTF8.GetBytes("There"));
"There"u8.ToArray());

EngineConfig cfg = new EngineConfig(_logger, new VariableCollection());
IProcessor processor = Processor.Create(cfg, o.Provider);
byte[] data = Encoding.UTF8.GetBytes("Hello \r\n There \r\n You");
byte[] data = "Hello \r\n There \r\n You"u8.ToArray();
Stream d = new MemoryStream(data);
MemoryStream result = new MemoryStream();
bool modified = processor.Run(d, result);
Expand All @@ -106,11 +106,11 @@ public void VerifyTrimWhitespaceNeitherDirection()
return 0;
},
true,
Encoding.UTF8.GetBytes("There"));
"There"u8.ToArray());

EngineConfig cfg = new EngineConfig(_logger, new VariableCollection());
IProcessor processor = Processor.Create(cfg, o.Provider);
byte[] data = Encoding.UTF8.GetBytes("Hello \r\n There \r\n You");
byte[] data = "Hello \r\n There \r\n You"u8.ToArray();
Stream d = new MemoryStream(data);
MemoryStream result = new MemoryStream();
bool modified = processor.Run(d, result);
Expand All @@ -131,11 +131,11 @@ public void VerifyConsumeWholeLine()
return 0;
},
true,
Encoding.UTF8.GetBytes("There"));
"There"u8.ToArray());

EngineConfig cfg = new EngineConfig(_logger, new VariableCollection());
IProcessor processor = Processor.Create(cfg, o.Provider);
byte[] data = Encoding.UTF8.GetBytes("Hello \r\n There \r\n You");
byte[] data = "Hello \r\n There \r\n You"u8.ToArray();
Stream d = new MemoryStream(data);
MemoryStream result = new MemoryStream();
bool modified = processor.Run(d, result);
Expand Down Expand Up @@ -164,11 +164,11 @@ public void VerifyWhitespaceHandlerConsumeWholeLine(bool trim, bool trimForward,
return 0;
},
true,
Encoding.UTF8.GetBytes("There"));
"There"u8.ToArray());

EngineConfig cfg = new EngineConfig(_logger, new VariableCollection());
IProcessor processor = Processor.Create(cfg, o.Provider);
byte[] data = Encoding.UTF8.GetBytes("Hello \r\n There \r\n You");
byte[] data = "Hello \r\n There \r\n You"u8.ToArray();
Stream d = new MemoryStream(data);
MemoryStream result = new MemoryStream();
bool modified = processor.Run(d, result);
Expand All @@ -193,11 +193,11 @@ public void VerifyWhitespaceHandlerTrim(bool trimForward, bool trimBackward)
return 0;
},
true,
Encoding.UTF8.GetBytes("There"));
"There"u8.ToArray());

EngineConfig cfg = new EngineConfig(_logger, new VariableCollection());
IProcessor processor = Processor.Create(cfg, o.Provider);
byte[] data = Encoding.UTF8.GetBytes("Hello \r\n There \r\n You");
byte[] data = "Hello \r\n There \r\n You"u8.ToArray();
Stream d = new MemoryStream(data);
MemoryStream result = new MemoryStream();
bool modified = processor.Run(d, result);
Expand All @@ -219,11 +219,11 @@ public void VerifyWhitespaceHandlerTrimForwardButNotBack()
return 0;
},
true,
Encoding.UTF8.GetBytes("There"));
"There"u8.ToArray());

EngineConfig cfg = new EngineConfig(_logger, new VariableCollection());
IProcessor processor = Processor.Create(cfg, o.Provider);
byte[] data = Encoding.UTF8.GetBytes("Hello \r\n There \r\n You");
byte[] data = "Hello \r\n There \r\n You"u8.ToArray();
Stream d = new MemoryStream(data);
MemoryStream result = new MemoryStream();
bool modified = processor.Run(d, result);
Expand All @@ -244,11 +244,11 @@ public void VerifyWhitespaceHandlerTrimBackButNotForward()
return 0;
},
true,
Encoding.UTF8.GetBytes("There"));
"There"u8.ToArray());

EngineConfig cfg = new EngineConfig(_logger, new VariableCollection());
IProcessor processor = Processor.Create(cfg, o.Provider);
byte[] data = Encoding.UTF8.GetBytes("Hello \r\n There \r\n You");
byte[] data = "Hello \r\n There \r\n You"u8.ToArray();
Stream d = new MemoryStream(data);
MemoryStream result = new MemoryStream();
bool modified = processor.Run(d, result);
Expand All @@ -269,11 +269,11 @@ public void VerifyWhitespaceHandlerTrimBackAndForward()
return 0;
},
true,
Encoding.UTF8.GetBytes("There"));
"There"u8.ToArray());

EngineConfig cfg = new EngineConfig(_logger, new VariableCollection());
IProcessor processor = Processor.Create(cfg, o.Provider);
byte[] data = Encoding.UTF8.GetBytes("Hello \r\n There \r\n You");
byte[] data = "Hello \r\n There \r\n You"u8.ToArray();
Stream d = new MemoryStream(data);
MemoryStream result = new MemoryStream();
bool modified = processor.Run(d, result);
Expand Down
2 changes: 1 addition & 1 deletion test/Microsoft.TemplateEngine.Core.UnitTests/LookaroundTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ public void TestReadAheadBreaksLookBehinds()

IOperationProvider[] operations =
{
new MockOperationProvider(new MockOperation(null, ReadaheadOneByte, true, Encoding.UTF8.GetBytes("foo"))),
new MockOperationProvider(new MockOperation(null, ReadaheadOneByte, true, "foo"u8.ToArray())),
new Replacement("bar".TokenConfigBuilder().OnlyIfAfter("foot"), "b", null, true)
};
EngineConfig cfg = new EngineConfig(_engineEnvironmentSettings.Host.Logger, VariableCollection.Root());
Expand Down
42 changes: 21 additions & 21 deletions test/Microsoft.TemplateEngine.Core.UnitTests/TokenTrieTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,21 +42,21 @@ public void VerifyTokenTrieLookArounds(string original, int checkPosition, int e
[Fact(DisplayName = nameof(VerifyTokenTrieAtBegin))]
public void VerifyTokenTrieAtBegin()
{
byte[] hello = Encoding.UTF8.GetBytes("hello");
byte[] helloBang = Encoding.UTF8.GetBytes("hello!");
byte[] hi = Encoding.UTF8.GetBytes("hi");
byte[] hello = "hello"u8.ToArray();
byte[] helloBang = "hello!"u8.ToArray();
byte[] hi = "hi"u8.ToArray();

TokenTrie t = new TokenTrie();
t.AddToken(hello);
t.AddToken(helloBang);
t.AddToken(hi);

byte[] source1 = Encoding.UTF8.GetBytes("hello there");
byte[] source2 = Encoding.UTF8.GetBytes("hello1 there");
byte[] source3 = Encoding.UTF8.GetBytes("hello! there");
byte[] source4 = Encoding.UTF8.GetBytes("hi there");
byte[] source5 = Encoding.UTF8.GetBytes("hi");
byte[] source6 = Encoding.UTF8.GetBytes("he");
byte[] source1 = "hello there"u8.ToArray();
byte[] source2 = "hello1 there"u8.ToArray();
byte[] source3 = "hello! there"u8.ToArray();
byte[] source4 = "hi there"u8.ToArray();
byte[] source5 = "hi"u8.ToArray();
byte[] source6 = "he"u8.ToArray();

int pos = 0;
Assert.True(t.GetOperation(source1, source1.Length, ref pos, out int token));
Expand Down Expand Up @@ -86,15 +86,15 @@ public void VerifyTokenTrieAtBegin()
[Fact(DisplayName = nameof(VerifyTokenTrieNotEnoughBufferLeft))]
public void VerifyTokenTrieNotEnoughBufferLeft()
{
byte[] hello = Encoding.UTF8.GetBytes("hello");
byte[] helloBang = Encoding.UTF8.GetBytes("hello!");
byte[] hello = "hello"u8.ToArray();
byte[] helloBang = "hello!"u8.ToArray();

TokenTrie t = new TokenTrie();
t.AddToken(hello);
t.AddToken(helloBang);

byte[] source1 = Encoding.UTF8.GetBytes("hi");
byte[] source2 = Encoding.UTF8.GetBytes(" hello");
byte[] source1 = "hi"u8.ToArray();
byte[] source2 = " hello"u8.ToArray();

int pos = 0;
Assert.False(t.GetOperation(source1, source1.Length, ref pos, out int token));
Expand All @@ -112,10 +112,10 @@ public void VerifyTokenTrieNotEnoughBufferLeft()
[Fact(DisplayName = nameof(VerifyTokenTrieCombine))]
public void VerifyTokenTrieCombine()
{
byte[] hello = Encoding.UTF8.GetBytes("hello");
byte[] helloBang = Encoding.UTF8.GetBytes("hello!");
byte[] hi = Encoding.UTF8.GetBytes("hi");
byte[] there = Encoding.UTF8.GetBytes("there!");
byte[] hello = "hello"u8.ToArray();
byte[] helloBang = "hello!"u8.ToArray();
byte[] hi = "hi"u8.ToArray();
byte[] there = "there!"u8.ToArray();

TokenTrie t = new TokenTrie();
t.AddToken(hello);
Expand All @@ -129,10 +129,10 @@ public void VerifyTokenTrieCombine()
combined.Append(t);
combined.Append(t2);

byte[] source1 = Encoding.UTF8.GetBytes("hello there");
byte[] source2 = Encoding.UTF8.GetBytes("hello! there");
byte[] source3 = Encoding.UTF8.GetBytes("hi there");
byte[] source4 = Encoding.UTF8.GetBytes("there!");
byte[] source1 = "hello there"u8.ToArray();
byte[] source2 = "hello! there"u8.ToArray();
byte[] source3 = "hi there"u8.ToArray();
byte[] source4 = "there!"u8.ToArray();

int pos = 0;
Assert.True(t.GetOperation(source1, source1.Length, ref pos, out int token));
Expand Down

0 comments on commit 9bcea72

Please sign in to comment.