Skip to content

Commit

Permalink
Include debian patch for unicode codepoints over 0xffff
Browse files Browse the repository at this point in the history
  • Loading branch information
qix committed Jul 13, 2016
1 parent 6d7a537 commit ca43af2
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 3 deletions.
6 changes: 5 additions & 1 deletion src/yaml/emitter.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,13 @@
__all__ = ['Emitter', 'EmitterError']

import six
import sys

from .error import YAMLError
from .events import *

has_ucs4 = sys.maxunicode > 0xffff

class EmitterError(YAMLError):
pass

Expand Down Expand Up @@ -708,7 +711,8 @@ def analyze_scalar(self, scalar):
line_breaks = True
if not (ch == '\n' or '\x20' <= ch <= '\x7E'):
if (ch == '\x85' or '\xA0' <= ch <= '\uD7FF'
or '\uE000' <= ch <= '\uFFFD') and ch != '\uFEFF':
or u'\uE000' <= ch <= u'\uFFFD'
or ((not has_ucs4) or (u'\U00010000' <= ch < u'\U0010ffff'))) and ch != u'\uFEFF':
unicode_characters = True
if not self.allow_unicode:
special_characters = True
Expand Down
12 changes: 10 additions & 2 deletions src/yaml/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,12 @@

from .error import YAMLError, Mark

import codecs, re
import codecs
import re
import six
import sys

has_ucs4 = sys.maxunicode > 0xffff

class ReaderError(YAMLError):

Expand Down Expand Up @@ -141,7 +145,11 @@ def determine_encoding(self):
self.encoding = 'utf-8'
self.update(1)

NON_PRINTABLE = re.compile('[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD]')
if has_ucs4:
NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]')
else:
NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD]')

def check_printable(self, data):
match = self.NON_PRINTABLE.search(data)
if match:
Expand Down

1 comment on commit ca43af2

@pombredanne
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi, do you have any unit tests for this by chance?

Please sign in to comment.