Tokenizer supports string split with multiple spaces.

See #69
alibaba · Apr 9, 2020 · 02b0ba5 · 02b0ba5
1 parent 0f9227f
commit 02b0ba5
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/core/src/main/java/com/alibaba/alink/operator/common/nlp/TokenizerMapper.java b/core/src/main/java/com/alibaba/alink/operator/common/nlp/TokenizerMapper.java
@@ -11,7 +11,7 @@
  * Transform all words into lower case, and split it by white space.
  */
 public class TokenizerMapper extends SISOMapper {
-	private static final String SPLIT_DELIMITER = "\\s";
+	private static final String SPLIT_DELIMITER = "\\s+";
 
 	public TokenizerMapper(TableSchema dataSchema, Params params) {
 		super(dataSchema, params);

diff --git a/core/src/test/java/com/alibaba/alink/operator/common/nlp/TokenizerMapperTest.java b/core/src/test/java/com/alibaba/alink/operator/common/nlp/TokenizerMapperTest.java
@@ -24,7 +24,7 @@ public void testDefault() throws Exception {
         TokenizerMapper mapper = new TokenizerMapper(schema, params);
 
         assertEquals(mapper.map(Row.of("This\tis  a unit test for mapper")).getField(0),
-            "this is  a unit test for mapper");
+            "this is a unit test for mapper");
         assertEquals(mapper.getOutputSchema(), schema);
     }