AtHeartEngineer/rails
1
1
Fork 0
mirror of https://github.com/github/rails.git synced 2026-01-10 07:08:08 -05:00
Code Issues Packages Projects Releases Wiki Activity

Simplify ActiveSupport::Multibyte and make it run on Ruby 1.9.

Browse Source 
* Unicode methods are now defined directly on Chars instead of a handler
* Updated Unicode database to Unicode 5.1.0
* Improved documentation
This commit is contained in:
Manfred Stienstra
2008-09-21 17:21:30 +02:00
parent 5f83e1844c
commit 22f75d539d
18 changed files with 1568 additions and 1556 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
activesupport/bin/generate_tables Executable file
View File

@@ -0,0 +1,147 @@
#!/usr/bin/env ruby
begin
$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
require 'active_support'
rescue IOError
end
require 'open-uri'
require 'tmpdir'
module ActiveSupport
module Multibyte
class UnicodeDatabase
def load; end
end
class UnicodeDatabaseGenerator
BASE_URI = "http://www.unicode.org/Public/#{ActiveSupport::Multibyte::UNICODE_VERSION}/ucd/"
SOURCES = {
:codepoints => BASE_URI + 'UnicodeData.txt',
:composition_exclusion => BASE_URI + 'CompositionExclusions.txt',
:grapheme_break_property => BASE_URI + 'auxiliary/GraphemeBreakProperty.txt',
:cp1252 => 'http://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1252.TXT'
}
def initialize
@ucd = UnicodeDatabase.new
default = Codepoint.new
default.combining_class = 0
default.uppercase_mapping = 0
default.lowercase_mapping = 0
@ucd.codepoints = Hash.new(default)
end
def parse_codepoints(line)
codepoint = Codepoint.new
raise "Could not parse input." unless line =~ /^
([0-9A-F]+); # code
([^;]+); # name
([A-Z]+); # general category
([0-9]+); # canonical combining class
([A-Z]+); # bidi class
(<([A-Z]*)>)? # decomposition type
((\ ?[0-9A-F]+)*); # decompomposition mapping
([0-9]*); # decimal digit
([0-9]*); # digit
([^;]*); # numeric
([YN]*); # bidi mirrored
([^;]*); # unicode 1.0 name
([^;]*); # iso comment
([0-9A-F]*); # simple uppercase mapping
([0-9A-F]*); # simple lowercase mapping
([0-9A-F]*)$/ix # simple titlecase mapping
codepoint.code = $1.hex
#codepoint.name = $2
#codepoint.category = $3
codepoint.combining_class = Integer($4)
#codepoint.bidi_class = $5
codepoint.decomp_type = $7
codepoint.decomp_mapping = ($8=='') ? nil : $8.split.collect { |element| element.hex }
#codepoint.bidi_mirrored = ($13=='Y') ? true : false
codepoint.uppercase_mapping = ($16=='') ? 0 : $16.hex
codepoint.lowercase_mapping = ($17=='') ? 0 : $17.hex
#codepoint.titlecase_mapping = ($18=='') ? nil : $18.hex
@ucd.codepoints[codepoint.code] = codepoint
end
def parse_grapheme_break_property(line)
if line =~ /^([0-9A-F\.]+)\s*;\s*([\w]+)\s*#/
type = $2.downcase.intern
@ucd.boundary[type] ||= []
if $1.include? '..'
parts = $1.split '..'
@ucd.boundary[type] << (parts[0].hex..parts[1].hex)
else
@ucd.boundary[type] << $1.hex
end
end
end
def parse_composition_exclusion(line)
if line =~ /^([0-9A-F]+)/i
@ucd.composition_exclusion << $1.hex
end
end
def parse_cp1252(line)
if line =~ /^([0-9A-Fx]+)\s([0-9A-Fx]+)/i
@ucd.cp1252[$1.hex] = $2.hex
end
end
def create_composition_map
@ucd.codepoints.each do |_, cp|
if !cp.nil? and cp.combining_class == 0 and cp.decomp_type.nil? and !cp.decomp_mapping.nil? and cp.decomp_mapping.length == 2 and @ucd.codepoints[cp.decomp_mapping[0]].combining_class == 0 and !@ucd.composition_exclusion.include?(cp.code)
@ucd.composition_map[cp.decomp_mapping[0]] ||= {}
@ucd.composition_map[cp.decomp_mapping[0]][cp.decomp_mapping[1]] = cp.code
end
end
end
def normalize_boundary_map
@ucd.boundary.each do |k,v|
if [:lf, :cr].include? k
@ucd.boundary[k] = v[0]
end
end
end
def parse
SOURCES.each do |type, url|
filename = File.join(Dir.tmpdir, "#{url.split('/').last}")
unless File.exist?(filename)
$stderr.puts "Downloading #{url.split('/').last}"
File.open(filename, 'wb') do |target|
open(url) do |source|
source.each_line { |line| target.write line }
end
end
end
File.open(filename) do |file|
file.each_line { |line| send "parse_#{type}".intern, line }
end
end
create_composition_map
normalize_boundary_map
end
def dump_to(filename)
File.open(filename, 'wb') do |f|
f.write Marshal.dump([@ucd.codepoints, @ucd.composition_exclusion, @ucd.composition_map, @ucd.boundary, @ucd.cp1252])
end
end
end
end
end
if __FILE__ == $0
filename = ActiveSupport::Multibyte::UnicodeDatabase.filename
generator = ActiveSupport::Multibyte::UnicodeDatabaseGenerator.new
generator.parse
print "Writing to: #{filename}"
generator.dump_to filename
puts " (#{File.size(filename)} bytes)"
end

6
activesupport/lib/active_support/core_ext/string.rb
View File

@@ -1,9 +1,11 @@
# encoding: utf-8
require 'active_support/core_ext/string/inflections'
require 'active_support/core_ext/string/conversions'
require 'active_support/core_ext/string/access'
require 'active_support/core_ext/string/starts_ends_with'
require 'active_support/core_ext/string/iterators'
require 'active_support/core_ext/string/unicode'
require 'active_support/core_ext/string/multibyte'
require 'active_support/core_ext/string/xchar'
require 'active_support/core_ext/string/filters'
require 'active_support/core_ext/string/behavior'
@@ -15,6 +17,6 @@ class String #:nodoc:
include ActiveSupport::CoreExtensions::String::Inflections
include ActiveSupport::CoreExtensions::String::StartsEndsWith
include ActiveSupport::CoreExtensions::String::Iterators
include ActiveSupport::CoreExtensions::String::Unicode
include ActiveSupport::CoreExtensions::String::Behavior
include ActiveSupport::CoreExtensions::String::Multibyte
end

81
activesupport/lib/active_support/core_ext/string/multibyte.rb Normal file
View File

@@ -0,0 +1,81 @@
# encoding: utf-8
module ActiveSupport #:nodoc:
module CoreExtensions #:nodoc:
module String #:nodoc:
# Implements multibyte methods for easier access to multibyte characters in a String instance.
module Multibyte
unless '1.9'.respond_to?(:force_encoding)
# +mb_chars+ is a multibyte safe proxy method for string methods.
#
# In Ruby 1.8 and older it creates and returns an instance of the ActiveSupport::Multibyte::Chars class which
# encapsulates the original string. A Unicode safe version of all the String methods are defined on this proxy
# class. If the proxy class doesn't respond to a certain method, it's forwarded to the encapsuled string.
#
# name = 'Claus Müller'
# name.reverse #=> "rell??M sualC"
# name.length #=> 13
#
# name.mb_chars.reverse.to_s #=> "rellüM sualC"
# name.mb_chars.length #=> 12
#
# In Ruby 1.9 and newer +mb_chars+ returns +self+ because String is (mostly) encoding aware so we don't need
# a proxy class any more. This means that +mb_chars+ makes it easier to write code that runs on multiple Ruby
# versions.
#
# == Method chaining
#
# All the methods on the Chars proxy which normally return a string will return a Chars object. This allows
# method chaining on the result of any of these methods.
#
# name.mb_chars.reverse.length #=> 12
#
# == Interoperability and configuration
#
# The Char object tries to be as interchangeable with String objects as possible: sorting and comparing between
# String and Char work like expected. The bang! methods change the internal string representation in the Chars
# object. Interoperability problems can be resolved easily with a +to_s+ call.
#
# For more information about the methods defined on the Chars proxy see ActiveSupport::Multibyte::Chars. For
# information about how to change the default Multibyte behaviour, see ActiveSupport::Multibyte.
def mb_chars
if ActiveSupport::Multibyte.proxy_class.wants?(self)
ActiveSupport::Multibyte.proxy_class.new(self)
else
self
end
end
# Returns true if the string has UTF-8 semantics (a String used for purely byte resources is unlikely to have
# them), returns false otherwise.
def is_utf8?
ActiveSupport::Multibyte::Chars.consumes?(self)
end
unless '1.8.7 and later'.respond_to?(:chars)
alias chars mb_chars
end
else
# In Ruby 1.9 and newer +mb_chars+ returns self. In Ruby 1.8 and older +mb_chars+ creates and returns an
# Unicode safe proxy for string operations, this makes it easier to write code that runs on multiple Ruby
# versions.
def mb_chars
self
end
# Returns true if the string has valid UTF-8 encoding.
def is_utf8?
case encoding
when Encoding::UTF_8
valid_encoding?
when Encoding::ASCII_8BIT, Encoding::US_ASCII
dup.force_encoding(Encoding::UTF_8).valid_encoding?
else
false
end
end
end
end
end
end
end

66
activesupport/lib/active_support/core_ext/string/unicode.rb
View File

@@ -1,66 +0,0 @@
module ActiveSupport #:nodoc:
module CoreExtensions #:nodoc:
module String #:nodoc:
# Define methods for handling unicode data.
module Unicode
def self.append_features(base)
if '1.8.7 and later'.respond_to?(:chars)
base.class_eval { remove_method :chars }
end
super
end
unless '1.9'.respond_to?(:force_encoding)
# +chars+ is a Unicode safe proxy for string methods. It creates and returns an instance of the
# ActiveSupport::Multibyte::Chars class which encapsulates the original string. A Unicode safe version of all
# the String methods are defined on this proxy class. Undefined methods are forwarded to String, so all of the
# string overrides can also be called through the +chars+ proxy.
#
# name = 'Claus Müller'
# name.reverse # => "rell??M sualC"
# name.length # => 13
#
# name.chars.reverse.to_s # => "rellüM sualC"
# name.chars.length # => 12
#
#
# All the methods on the chars proxy which normally return a string will return a Chars object. This allows
# method chaining on the result of any of these methods.
#
# name.chars.reverse.length # => 12
#
# The Char object tries to be as interchangeable with String objects as possible: sorting and comparing between
# String and Char work like expected. The bang! methods change the internal string representation in the Chars
# object. Interoperability problems can be resolved easily with a +to_s+ call.
#
# For more information about the methods defined on the Chars proxy see ActiveSupport::Multibyte::Chars and
# ActiveSupport::Multibyte::Handlers::UTF8Handler.
def chars
ActiveSupport::Multibyte::Chars.new(self)
end
# Returns true if the string has UTF-8 semantics (a String used for purely byte resources is unlikely to have
# them), returns false otherwise.
def is_utf8?
ActiveSupport::Multibyte::Handlers::UTF8Handler.consumes?(self)
end
else
def chars #:nodoc:
self
end
def is_utf8? #:nodoc:
case encoding
when Encoding::UTF_8
valid_encoding?
when Encoding::ASCII_8BIT
dup.force_encoding('UTF-8').valid_encoding?
else
false
end
end
end
end
end
end
end

38
activesupport/lib/active_support/multibyte.rb
View File

@@ -1,9 +1,33 @@
module ActiveSupport
module Multibyte #:nodoc:
DEFAULT_NORMALIZATION_FORM = :kc
NORMALIZATIONS_FORMS = [:c, :kc, :d, :kd]
UNICODE_VERSION = '5.0.0'
end
end
# encoding: utf-8
require 'active_support/multibyte/chars'
require 'active_support/multibyte/exceptions'
require 'active_support/multibyte/unicode_database'
module ActiveSupport #:nodoc:
module Multibyte #:nodoc:
# A list of all available normalization forms. See http://www.unicode.org/reports/tr15/tr15-29.html for more
# information about normalization.
NORMALIZATIONS_FORMS = [:c, :kc, :d, :kd]
# The Unicode version that is supported by the implementation
UNICODE_VERSION = '5.1.0'
# The default normalization used for operations that require normalization. It can be set to any of the
# normalizations in NORMALIZATIONS_FORMS.
#
# Example:
# ActiveSupport::Multibyte.default_normalization_form = :c
mattr_accessor :default_normalization_form
self.default_normalization_form = :kc
# The proxy class returned when calling mb_chars. You can use this accessor to configure your own proxy
# class so you can support other encodings. See the ActiveSupport::Multibyte::Chars implementation for
# an example how to do this.
#
# Example:
# ActiveSupport::Multibyte.proxy_class = CharsForUTF32
mattr_accessor :proxy_class
self.proxy_class = ActiveSupport::Multibyte::Chars
end
end

784
activesupport/lib/active_support/multibyte/chars.rb
View File

@@ -1,142 +1,670 @@
require 'active_support/multibyte/handlers/utf8_handler'
require 'active_support/multibyte/handlers/passthru_handler'
# encoding: utf-8
# Encapsulates all the functionality related to the Chars proxy.
module ActiveSupport::Multibyte #:nodoc:
# Chars enables you to work transparently with multibyte encodings in the Ruby String class without having extensive
# knowledge about the encoding. A Chars object accepts a string upon initialization and proxies String methods in an
# encoding safe manner. All the normal String methods are also implemented on the proxy.
#
# String methods are proxied through the Chars object, and can be accessed through the +chars+ method. Methods
# which would normally return a String object now return a Chars object so methods can be chained.
#
# "The Perfect String ".chars.downcase.strip.normalize # => "the perfect string"
#
# Chars objects are perfectly interchangeable with String objects as long as no explicit class checks are made.
# If certain methods do explicitly check the class, call +to_s+ before you pass chars objects to them.
#
# bad.explicit_checking_method "T".chars.downcase.to_s
#
# The actual operations on the string are delegated to handlers. Theoretically handlers can be implemented for
# any encoding, but the default handler handles UTF-8. This handler is set during initialization, if you want to
# use you own handler, you can set it on the Chars class. Look at the UTF8Handler source for an example how to
# implement your own handler. If you your own handler to work on anything but UTF-8 you probably also
# want to override Chars#handler.
#
# ActiveSupport::Multibyte::Chars.handler = MyHandler
#
# Note that a few methods are defined on Chars instead of the handler because they are defined on Object or Kernel
# and method_missing can't catch them.
class Chars
attr_reader :string # The contained string
alias_method :to_s, :string
include Comparable
# The magic method to make String and Chars comparable
def to_str
# Using any other ways of overriding the String itself will lead you all the way from infinite loops to
# core dumps. Don't go there.
@string
end
module ActiveSupport #:nodoc:
module Multibyte #:nodoc:
# Chars enables you to work transparently with multibyte encodings in the Ruby String class without having extensive
# knowledge about the encoding. A Chars object accepts a string upon initialization and proxies String methods in an
# encoding safe manner. All the normal String methods are also implemented on the proxy.
#
# String methods are proxied through the Chars object, and can be accessed through the +mb_chars+ method. Methods
# which would normally return a String object now return a Chars object so methods can be chained.
#
# "The Perfect String ".chars.downcase.strip.normalize #=> "the perfect string"
#
# Chars objects are perfectly interchangeable with String objects as long as no explicit class checks are made.
# If certain methods do explicitly check the class, call +to_s+ before you pass chars objects to them.
#
# bad.explicit_checking_method "T".chars.downcase.to_s
#
# The default Chars implementation assumes that the encoding of the string is UTF-8, if you want to handle different
# encodings you can write your own multibyte string handler and configure it through
# ActiveSupport::Multibyte.proxy_class.
#
# class CharsForUTF32
# def size
# @wrapped_string.size / 4
# end
#
# def self.accepts?(string)
# string.length % 4 == 0
# end
# end
#
# ActiveSupport::Multibyte.proxy_class = CharsForUTF32
class Chars
# Hangul character boundaries and properties
HANGUL_SBASE = 0xAC00
HANGUL_LBASE = 0x1100
HANGUL_VBASE = 0x1161
HANGUL_TBASE = 0x11A7
HANGUL_LCOUNT = 19
HANGUL_VCOUNT = 21
HANGUL_TCOUNT = 28
HANGUL_NCOUNT = HANGUL_VCOUNT * HANGUL_TCOUNT
HANGUL_SCOUNT = 11172
HANGUL_SLAST = HANGUL_SBASE + HANGUL_SCOUNT
HANGUL_JAMO_FIRST = 0x1100
HANGUL_JAMO_LAST = 0x11FF
# Make duck-typing with String possible
def respond_to?(method, include_priv = false)
super || @string.respond_to?(method, include_priv) ||
handler.respond_to?(method, include_priv) ||
(method.to_s =~ /(.*)!/ && handler.respond_to?($1, include_priv)) ||
false
end
# All the unicode whitespace
UNICODE_WHITESPACE = [
(0x0009..0x000D).to_a, # White_Space # Cc [5] <control-0009>..<control-000D>
0x0020, # White_Space # Zs SPACE
0x0085, # White_Space # Cc <control-0085>
0x00A0, # White_Space # Zs NO-BREAK SPACE
0x1680, # White_Space # Zs OGHAM SPACE MARK
0x180E, # White_Space # Zs MONGOLIAN VOWEL SEPARATOR
(0x2000..0x200A).to_a, # White_Space # Zs [11] EN QUAD..HAIR SPACE
0x2028, # White_Space # Zl LINE SEPARATOR
0x2029, # White_Space # Zp PARAGRAPH SEPARATOR
0x202F, # White_Space # Zs NARROW NO-BREAK SPACE
0x205F, # White_Space # Zs MEDIUM MATHEMATICAL SPACE
0x3000, # White_Space # Zs IDEOGRAPHIC SPACE
].flatten.freeze
# Enable more predictable duck-typing on String-like classes. See Object#acts_like?.
def acts_like_string?
true
end
# BOM (byte order mark) can also be seen as whitespace, it's a non-rendering character used to distinguish
# between little and big endian. This is not an issue in utf-8, so it must be ignored.
UNICODE_LEADERS_AND_TRAILERS = UNICODE_WHITESPACE + [65279] # ZERO-WIDTH NO-BREAK SPACE aka BOM
# Create a new Chars instance.
def initialize(str)
@string = str.respond_to?(:string) ? str.string : str
end
# Returns -1, 0 or +1 depending on whether the Chars object is to be sorted before, equal or after the
# object on the right side of the operation. It accepts any object that implements +to_s+. See String.<=>
# for more details.
def <=>(other); @string <=> other.to_s; end
# Works just like String#split, with the exception that the items in the resulting list are Chars
# instances instead of String. This makes chaining methods easier.
def split(*args)
@string.split(*args).map { |i| i.chars }
end
# Gsub works exactly the same as gsub on a normal string.
def gsub(*a, &b); @string.gsub(*a, &b).chars; end
# Like String.=~ only it returns the character offset (in codepoints) instead of the byte offset.
def =~(other)
handler.translate_offset(@string, @string =~ other)
end
# Try to forward all undefined methods to the handler, when a method is not defined on the handler, send it to
# the contained string. Method_missing is also responsible for making the bang! methods destructive.
def method_missing(m, *a, &b)
begin
# Simulate methods with a ! at the end because we can't touch the enclosed string from the handlers.
if m.to_s =~ /^(.*)\!$/ && handler.respond_to?($1)
result = handler.send($1, @string, *a, &b)
if result == @string
result = nil
else
@string.replace result
end
elsif handler.respond_to?(m)
result = handler.send(m, @string, *a, &b)
else
result = @string.send(m, *a, &b)
# Returns a regular expression pattern that matches the passed Unicode codepoints
def self.codepoints_to_pattern(array_of_codepoints) #:nodoc:
array_of_codepoints.collect{ |e| [e].pack 'U*' }.join('|')
end
UNICODE_TRAILERS_PAT = /(#{codepoints_to_pattern(UNICODE_LEADERS_AND_TRAILERS)})+\Z/
UNICODE_LEADERS_PAT = /\A(#{codepoints_to_pattern(UNICODE_LEADERS_AND_TRAILERS)})+/
# Borrowed from the Kconv library by Shinji KONO - (also as seen on the W3C site)
UTF8_PAT = /\A(?:
[\x00-\x7f] |
[\xc2-\xdf] [\x80-\xbf] |
\xe0 [\xa0-\xbf] [\x80-\xbf] |
[\xe1-\xef] [\x80-\xbf] [\x80-\xbf] |
\xf0 [\x90-\xbf] [\x80-\xbf] [\x80-\xbf] |
[\xf1-\xf3] [\x80-\xbf] [\x80-\xbf] [\x80-\xbf] |
\xf4 [\x80-\x8f] [\x80-\xbf] [\x80-\xbf]
)*\z/xn
attr_reader :wrapped_string
alias to_s wrapped_string
alias to_str wrapped_string
# Creates a new Chars instance. +string+ is the wrapped string.
if '1.9'.respond_to?(:force_encoding)
def initialize(string)
@wrapped_string = string
@wrapped_string.force_encoding(Encoding::UTF_8) unless @wrapped_string.frozen?
end
else
def initialize(string)
@wrapped_string = string
end
end
# Forward all undefined methods to the wrapped string.
def method_missing(method, *args, &block)
if method.to_s =~ /!$/
@wrapped_string.__send__(method, *args, &block)
self
else
chars(@wrapped_string.__send__(method, *args, &block))
end
rescue Handlers::EncodingError
@string.replace handler.tidy_bytes(@string)
retry
end
if result.kind_of?(String)
result.chars
else
result
# Returns +true+ if _obj_ responds to the given method. Private methods are included in the search
# only if the optional second parameter evaluates to +true+.
def respond_to?(method, include_private=false)
super || @wrapped_string.respond_to?(method, include_private) || false
end
end
# Set the handler class for the Char objects.
def self.handler=(klass)
@@handler = klass
end
# Returns the proper handler for the contained string depending on $KCODE and the encoding of the string. This
# method is used internally to always redirect messages to the proper classes depending on the context.
def handler
if utf8_pragma?
@@handler
else
ActiveSupport::Multibyte::Handlers::PassthruHandler
# Enable more predictable duck-typing on String-like classes. See Object#acts_like?.
def acts_like_string?
true
end
end
private
# Returns +true+ if the Chars class can and should act as a proxy for the string +string+. Returns
# +false+ otherwise.
def self.wants?(string)
RUBY_VERSION < '1.9' && $KCODE == 'UTF8' && consumes?(string)
end
# Returns +true+ when the proxy class can handle the string. Returns +false+ otherwise.
def self.consumes?(string)
# Unpack is a little bit faster than regular expressions.
string.unpack('U*')
true
rescue ArgumentError
false
end
include Comparable
# Returns -1, 0 or +1 depending on whether the Chars object is to be sorted before, equal or after the
# object on the right side of the operation. It accepts any object that implements +to_s+. See String.<=>
# for more details.
#
# Example:
# 'é'.mb_chars <=> 'ü'.mb_chars #=> -1
def <=>(other)
@wrapped_string <=> other.to_s
end
# Returns a new Chars object containing the other object concatenated to the string.
#
# Example:
# ('Café'.mb_chars + ' périferôl').to_s #=> "Café périferôl"
def +(other)
self << other
end
# Like String.=~ only it returns the character offset (in codepoints) instead of the byte offset.
#
# Example:
# 'Café périferôl'.mb_chars =~ /ô/ #=> 12
def =~(other)
translate_offset(@wrapped_string =~ other)
end
# Works just like String#split, with the exception that the items in the resulting list are Chars
# instances instead of String. This makes chaining methods easier.
#
# Example:
# 'Café périferôl'.mb_chars.split(/é/).map { |part| part.upcase.to_s } #=> ["CAF", " P", "RIFERÔL"]
def split(*args)
@wrapped_string.split(*args).map { |i| i.mb_chars }
end
# Inserts the passed string at specified codepoint offsets
#
# Example:
# 'Café'.mb_chars.insert(4, ' périferôl').to_s #=> "Café périferôl"
def insert(offset, fragment)
unpacked = self.class.u_unpack(@wrapped_string)
unless offset > unpacked.length
@wrapped_string.replace(
self.class.u_unpack(@wrapped_string).insert(offset, *self.class.u_unpack(fragment)).pack('U*')
)
else
raise IndexError, "index #{offset} out of string"
end
self
end
# Returns true if contained string contains +other+. Returns false otherwise.
#
# Example:
# 'Café'.mb_chars.include?('é') #=> true
def include?(other)
# We have to redefine this method because Enumerable defines it.
@wrapped_string.include?(other)
end
# Returns the position of the passed argument in the string, counting in codepoints
#
# Example:
# 'Café périferôl'.mb_chars.index('ô') #=> 12
def index(*args)
index = @wrapped_string.index(*args)
index ? (self.class.u_unpack(@wrapped_string.slice(0...index)).size) : nil
end
# Works just like the indexed replace method on string, except instead of byte offsets you specify
# character offsets.
#
# Example:
#
# s = "Müller"
# s.chars[2] = "e" # Replace character with offset 2
# s
# #=> "Müeler"
#
# s = "Müller"
# s.chars[1, 2] = "ö" # Replace 2 characters at character offset 1
# s
# #=> "Möler"
def []=(*args)
replace_by = args.pop
# Indexed replace with regular expressions already works
if args.first.is_a?(Regexp)
@wrapped_string[*args] = replace_by
else
result = self.class.u_unpack(@wrapped_string)
if args[0].is_a?(Fixnum)
raise IndexError, "index #{args[0]} out of string" if args[0] >= result.length
min = args[0]
max = args[1].nil? ? min : (min + args[1] - 1)
range = Range.new(min, max)
replace_by = [replace_by].pack('U') if replace_by.is_a?(Fixnum)
elsif args.first.is_a?(Range)
raise RangeError, "#{args[0]} out of range" if args[0].min >= result.length
range = args[0]
else
needle = args[0].to_s
min = index(needle)
max = min + self.class.u_unpack(needle).length - 1
range = Range.new(min, max)
end
result[range] = self.class.u_unpack(replace_by)
@wrapped_string.replace(result.pack('U*'))
end
self
end
# Works just like String#rjust, only integer specifies characters instead of bytes.
#
# Example:
#
# "¾ cup".chars.rjust(8).to_s
# #=> " ¾ cup"
#
# "¾ cup".chars.rjust(8, " ").to_s # Use non-breaking whitespace
# #=> "   ¾ cup"
def rjust(integer, padstr=' ')
justify(integer, :right, padstr)
end
# Works just like String#ljust, only integer specifies characters instead of bytes.
#
# Example:
#
# "¾ cup".chars.rjust(8).to_s
# #=> "¾ cup "
#
# "¾ cup".chars.rjust(8, " ").to_s # Use non-breaking whitespace
# #=> "¾ cup   "
def ljust(integer, padstr=' ')
justify(integer, :left, padstr)
end
# Works just like String#center, only integer specifies characters instead of bytes.
#
# Example:
#
# "¾ cup".chars.center(8).to_s
# #=> " ¾ cup "
#
# "¾ cup".chars.center(8, " ").to_s # Use non-breaking whitespace
# #=> " ¾ cup  "
def center(integer, padstr=' ')
justify(integer, :center, padstr)
end
# Strips entire range of Unicode whitespace from the right of the string.
def rstrip
chars(@wrapped_string.gsub(UNICODE_TRAILERS_PAT, ''))
end
# +utf8_pragma+ checks if it can send this string to the handlers. It makes sure @string isn't nil and $KCODE is
# set to 'UTF8'.
def utf8_pragma?
!@string.nil? && ($KCODE == 'UTF8')
# Strips entire range of Unicode whitespace from the left of the string.
def lstrip
chars(@wrapped_string.gsub(UNICODE_LEADERS_PAT, ''))
end
# Strips entire range of Unicode whitespace from the right and left of the string.
def strip
rstrip.lstrip
end
# Returns the number of codepoints in the string
def size
self.class.u_unpack(@wrapped_string).size
end
alias_method :length, :size
# Reverses all characters in the string
#
# Example:
# 'Café'.mb_chars.reverse.to_s #=> 'éfaC'
def reverse
chars(self.class.u_unpack(@wrapped_string).reverse.pack('U*'))
end
# Implements Unicode-aware slice with codepoints. Slicing on one point returns the codepoints for that
# character.
#
# Example:
# 'こにちわ'.mb_chars.slice(2..3).to_s #=> "ちわ"
def slice(*args)
if args.size > 2
raise ArgumentError, "wrong number of arguments (#{args.size} for 1)" # Do as if we were native
elsif (args.size == 2 && !(args.first.is_a?(Numeric) || args.first.is_a?(Regexp)))
raise TypeError, "cannot convert #{args.first.class} into Integer" # Do as if we were native
elsif (args.size == 2 && !args[1].is_a?(Numeric))
raise TypeError, "cannot convert #{args[1].class} into Integer" # Do as if we were native
elsif args[0].kind_of? Range
cps = self.class.u_unpack(@wrapped_string).slice(*args)
result = cps.nil? ? nil : cps.pack('U*')
elsif args[0].kind_of? Regexp
result = @wrapped_string.slice(*args)
elsif args.size == 1 && args[0].kind_of?(Numeric)
character = self.class.u_unpack(@wrapped_string)[args[0]]
result = character.nil? ? nil : [character].pack('U')
else
result = self.class.u_unpack(@wrapped_string).slice(*args).pack('U*')
end
result.nil? ? nil : chars(result)
end
alias_method :[], :slice
# Convert characters in the string to uppercase
#
# Example:
# 'Laurent, òu sont les tests?'.mb_chars.upcase.to_s #=> "LAURENT, ÒU SONT LES TESTS?"
def upcase
apply_mapping :uppercase_mapping
end
# Convert characters in the string to lowercase
#
# Example:
# 'VĚDA A VÝZKUM'.mb_chars.downcase.to_s #=> "věda a výzkum"
def downcase
apply_mapping :lowercase_mapping
end
# Converts the first character to uppercase and the remainder to lowercase
#
# Example:
# 'über'.mb_chars.capitalize.to_s #=> "Über"
def capitalize
(slice(0) || '').upcase + (slice(1..-1) || '').downcase
end
# Returns the KC normalization of the string by default. NFKC is considered the best normalization form for
# passing strings to databases and validations.
#
# * <tt>str</tt> - The string to perform normalization on.
# * <tt>form</tt> - The form you want to normalize in. Should be one of the following:
# <tt>:c</tt>, <tt>:kc</tt>, <tt>:d</tt>, or <tt>:kd</tt>. Default is
# ActiveSupport::Multibyte.default_normalization_form
def normalize(form=ActiveSupport::Multibyte.default_normalization_form)
# See http://www.unicode.org/reports/tr15, Table 1
codepoints = self.class.u_unpack(@wrapped_string)
chars(case form
when :d
self.class.reorder_characters(self.class.decompose_codepoints(:canonical, codepoints))
when :c
self.class.compose_codepoints(self.class.reorder_characters(self.class.decompose_codepoints(:canonical, codepoints)))
when :kd
self.class.reorder_characters(self.class.decompose_codepoints(:compatability, codepoints))
when :kc
self.class.compose_codepoints(self.class.reorder_characters(self.class.decompose_codepoints(:compatability, codepoints)))
else
raise ArgumentError, "#{form} is not a valid normalization variant", caller
end.pack('U*'))
end
# Performs canonical decomposition on all the characters.
#
# Example:
# 'é'.length #=> 2
# 'é'.mb_chars.decompose.to_s.length #=> 3
def decompose
chars(self.class.decompose_codepoints(:canonical, self.class.u_unpack(@wrapped_string)).pack('U*'))
end
# Performs composition on all the characters.
#
# Example:
# 'é'.length #=> 3
# 'é'.mb_chars.compose.to_s.length #=> 2
def compose
chars(self.class.compose_codepoints(self.class.u_unpack(@wrapped_string)).pack('U*'))
end
# Returns the number of grapheme clusters in the string.
#
# Example:
# 'क्षि'.mb_chars.length #=> 4
# 'क्षि'.mb_chars.g_length #=> 3
def g_length
self.class.g_unpack(@wrapped_string).length
end
def tidy_bytes
chars(self.class.tidy_bytes(@wrapped_string))
end
%w(lstrip rstrip strip reverse upcase downcase slice tidy_bytes capitalize).each do |method|
define_method("#{method}!") do |*args|
unless args.nil?
@wrapped_string = send(method, *args).to_s
else
@wrapped_string = send(method).to_s
end
self
end
end
class << self
# Unpack the string at codepoints boundaries
def u_unpack(str)
begin
str.unpack 'U*'
rescue ArgumentError
raise EncodingError.new('malformed UTF-8 character')
end
end
# Detect whether the codepoint is in a certain character class. Primarily used by the
# grapheme cluster support.
def in_char_class?(codepoint, classes)
classes.detect { |c| UCD.boundary[c] === codepoint } ? true : false
end
# Unpack the string at grapheme boundaries
def g_unpack(str)
codepoints = u_unpack(str)
unpacked = []
pos = 0
marker = 0
eoc = codepoints.length
while(pos < eoc)
pos += 1
previous = codepoints[pos-1]
current = codepoints[pos]
if (
# CR X LF
one = ( previous == UCD.boundary[:cr] and current == UCD.boundary[:lf] ) or
# L X (L|V|LV|LVT)
two = ( UCD.boundary[:l] === previous and in_char_class?(current, [:l,:v,:lv,:lvt]) ) or
# (LV|V) X (V|T)
three = ( in_char_class?(previous, [:lv,:v]) and in_char_class?(current, [:v,:t]) ) or
# (LVT|T) X (T)
four = ( in_char_class?(previous, [:lvt,:t]) and UCD.boundary[:t] === current ) or
# X Extend
five = (UCD.boundary[:extend] === current)
)
else
unpacked << codepoints[marker..pos-1]
marker = pos
end
end
unpacked
end
# Reverse operation of g_unpack
def g_pack(unpacked)
(unpacked.flatten).pack('U*')
end
# Generates a padding string of a certain size.
def padding(padsize, padstr=' ')
if padsize != 0
new(padstr * ((padsize / u_unpack(padstr).size) + 1)).slice(0, padsize)
else
''
end
end
# Re-order codepoints so the string becomes canonical
def reorder_characters(codepoints)
length = codepoints.length- 1
pos = 0
while pos < length do
cp1, cp2 = UCD.codepoints[codepoints[pos]], UCD.codepoints[codepoints[pos+1]]
if (cp1.combining_class > cp2.combining_class) && (cp2.combining_class > 0)
codepoints[pos..pos+1] = cp2.code, cp1.code
pos += (pos > 0 ? -1 : 1)
else
pos += 1
end
end
codepoints
end
# Decompose composed characters to the decomposed form
def decompose_codepoints(type, codepoints)
codepoints.inject([]) do |decomposed, cp|
# if it's a hangul syllable starter character
if HANGUL_SBASE <= cp and cp < HANGUL_SLAST
sindex = cp - HANGUL_SBASE
ncp = [] # new codepoints
ncp << HANGUL_LBASE + sindex / HANGUL_NCOUNT
ncp << HANGUL_VBASE + (sindex % HANGUL_NCOUNT) / HANGUL_TCOUNT
tindex = sindex % HANGUL_TCOUNT
ncp << (HANGUL_TBASE + tindex) unless tindex == 0
decomposed.concat ncp
# if the codepoint is decomposable in with the current decomposition type
elsif (ncp = UCD.codepoints[cp].decomp_mapping) and (!UCD.codepoints[cp].decomp_type || type == :compatability)
decomposed.concat decompose_codepoints(type, ncp.dup)
else
decomposed << cp
end
end
end
# Compose decomposed characters to the composed form
def compose_codepoints(codepoints)
pos = 0
eoa = codepoints.length - 1
starter_pos = 0
starter_char = codepoints[0]
previous_combining_class = -1
while pos < eoa
pos += 1
lindex = starter_char - HANGUL_LBASE
# -- Hangul
if 0 <= lindex and lindex < HANGUL_LCOUNT
vindex = codepoints[starter_pos+1] - HANGUL_VBASE rescue vindex = -1
if 0 <= vindex and vindex < HANGUL_VCOUNT
tindex = codepoints[starter_pos+2] - HANGUL_TBASE rescue tindex = -1
if 0 <= tindex and tindex < HANGUL_TCOUNT
j = starter_pos + 2
eoa -= 2
else
tindex = 0
j = starter_pos + 1
eoa -= 1
end
codepoints[starter_pos..j] = (lindex * HANGUL_VCOUNT + vindex) * HANGUL_TCOUNT + tindex + HANGUL_SBASE
end
starter_pos += 1
starter_char = codepoints[starter_pos]
# -- Other characters
else
current_char = codepoints[pos]
current = UCD.codepoints[current_char]
if current.combining_class > previous_combining_class
if ref = UCD.composition_map[starter_char]
composition = ref[current_char]
else
composition = nil
end
unless composition.nil?
codepoints[starter_pos] = composition
starter_char = composition
codepoints.delete_at pos
eoa -= 1
pos -= 1
previous_combining_class = -1
else
previous_combining_class = current.combining_class
end
else
previous_combining_class = current.combining_class
end
if current.combining_class == 0
starter_pos = pos
starter_char = codepoints[pos]
end
end
end
codepoints
end
# Replaces all the non-UTF-8 bytes by their iso-8859-1 or cp1252 equivalent resulting in a valid UTF-8 string
def tidy_bytes(str)
str.split(//u).map do |c|
if !UTF8_PAT.match(c)
n = c.unpack('C')[0]
n < 128 ? n.chr :
n < 160 ? [UCD.cp1252[n] || n].pack('U') :
n < 192 ? "\xC2" + n.chr : "\xC3" + (n-64).chr
else
c
end
end.join
end
end
protected
# Translate a byte offset in the wrapped string to a character offset by looking for the character boundary
def translate_offset(byte_offset)
return nil if byte_offset.nil?
return 0 if @wrapped_string == ''
chunk = @wrapped_string[0..byte_offset]
begin
begin
chunk.unpack('U*').length - 1
rescue ArgumentError => e
chunk = @wrapped_string[0..(byte_offset+=1)]
# Stop retrying at the end of the string
raise e unless byte_offset < chunk.length
# We damaged a character, retry
retry
end
# Catch the ArgumentError so we can throw our own
rescue ArgumentError
raise EncodingError, 'malformed UTF-8 character'
end
end
# Justifies a string in a certain way. Valid values for <tt>way</tt> are <tt>:right</tt>, <tt>:left</tt> and
# <tt>:center</tt>.
def justify(integer, way, padstr=' ')
raise ArgumentError, "zero width padding" if padstr.length == 0
padsize = integer - size
padsize = padsize > 0 ? padsize : 0
case way
when :right
result = @wrapped_string.dup.insert(0, self.class.padding(padsize, padstr))
when :left
result = @wrapped_string.dup.insert(-1, self.class.padding(padsize, padstr))
when :center
lpad = self.class.padding((padsize / 2.0).floor, padstr)
rpad = self.class.padding((padsize / 2.0).ceil, padstr)
result = @wrapped_string.dup.insert(0, lpad).insert(-1, rpad)
end
chars(result)
end
# Map codepoints to one of it's attributes.
def apply_mapping(mapping)
chars(self.class.u_unpack(@wrapped_string).map do |codepoint|
cp = UCD.codepoints[codepoint]
if cp and (ncp = cp.send(mapping)) and ncp > 0
ncp
else
codepoint
end
end.pack('U*'))
end
# Creates a new instance
def chars(str)
self.class.new(str)
end
end
end
end
# When we can load the utf8proc library, override normalization with the faster methods
begin
require 'utf8proc_native'
require 'active_support/multibyte/handlers/utf8_handler_proc'
ActiveSupport::Multibyte::Chars.handler = ActiveSupport::Multibyte::Handlers::UTF8HandlerProc
rescue LoadError
ActiveSupport::Multibyte::Chars.handler = ActiveSupport::Multibyte::Handlers::UTF8Handler
end
end

7
activesupport/lib/active_support/multibyte/exceptions.rb Normal file
View File

@@ -0,0 +1,7 @@
# encoding: utf-8
module ActiveSupport #:nodoc:
module Multibyte #:nodoc:
class EncodingError < StandardError; end
end
end

149
activesupport/lib/active_support/multibyte/generators/generate_tables.rb
View File

@@ -1,149 +0,0 @@
#!/usr/bin/env ruby
begin
require File.dirname(__FILE__) + '/../../../active_support'
rescue IOError
end
require 'open-uri'
require 'tmpdir'
module ActiveSupport::Multibyte::Handlers #:nodoc:
class UnicodeDatabase #:nodoc:
def self.load
[Hash.new(Codepoint.new),[],{},{}]
end
end
class UnicodeTableGenerator #:nodoc:
BASE_URI = "http://www.unicode.org/Public/#{ActiveSupport::Multibyte::UNICODE_VERSION}/ucd/"
SOURCES = {
:codepoints => BASE_URI + 'UnicodeData.txt',
:composition_exclusion => BASE_URI + 'CompositionExclusions.txt',
:grapheme_break_property => BASE_URI + 'auxiliary/GraphemeBreakProperty.txt',
:cp1252 => 'http://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1252.TXT'
}
def initialize
@ucd = UnicodeDatabase.new
default = Codepoint.new
default.combining_class = 0
default.uppercase_mapping = 0
default.lowercase_mapping = 0
@ucd.codepoints = Hash.new(default)
@ucd.composition_exclusion = []
@ucd.composition_map = {}
@ucd.boundary = {}
@ucd.cp1252 = {}
end
def parse_codepoints(line)
codepoint = Codepoint.new
raise "Could not parse input." unless line =~ /^
([0-9A-F]+); # code
([^;]+); # name
([A-Z]+); # general category
([0-9]+); # canonical combining class
([A-Z]+); # bidi class
(<([A-Z]*)>)? # decomposition type
((\ ?[0-9A-F]+)*); # decompomposition mapping
([0-9]*); # decimal digit
([0-9]*); # digit
([^;]*); # numeric
([YN]*); # bidi mirrored
([^;]*); # unicode 1.0 name
([^;]*); # iso comment
([0-9A-F]*); # simple uppercase mapping
([0-9A-F]*); # simple lowercase mapping
([0-9A-F]*)$/ix # simple titlecase mapping
codepoint.code = $1.hex
#codepoint.name = $2
#codepoint.category = $3
codepoint.combining_class = Integer($4)
#codepoint.bidi_class = $5
codepoint.decomp_type = $7
codepoint.decomp_mapping = ($8=='') ? nil : $8.split.collect { |element| element.hex }
#codepoint.bidi_mirrored = ($13=='Y') ? true : false
codepoint.uppercase_mapping = ($16=='') ? 0 : $16.hex
codepoint.lowercase_mapping = ($17=='') ? 0 : $17.hex
#codepoint.titlecase_mapping = ($18=='') ? nil : $18.hex
@ucd.codepoints[codepoint.code] = codepoint
end
def parse_grapheme_break_property(line)
if line =~ /^([0-9A-F\.]+)\s*;\s*([\w]+)\s*#/
type = $2.downcase.intern
@ucd.boundary[type] ||= []
if $1.include? '..'
parts = $1.split '..'
@ucd.boundary[type] << (parts[0].hex..parts[1].hex)
else
@ucd.boundary[type] << $1.hex
end
end
end
def parse_composition_exclusion(line)
if line =~ /^([0-9A-F]+)/i
@ucd.composition_exclusion << $1.hex
end
end
def parse_cp1252(line)
if line =~ /^([0-9A-Fx]+)\s([0-9A-Fx]+)/i
@ucd.cp1252[$1.hex] = $2.hex
end
end
def create_composition_map
@ucd.codepoints.each do |_, cp|
if !cp.nil? and cp.combining_class == 0 and cp.decomp_type.nil? and !cp.decomp_mapping.nil? and cp.decomp_mapping.length == 2 and @ucd[cp.decomp_mapping[0]].combining_class == 0 and !@ucd.composition_exclusion.include?(cp.code)
@ucd.composition_map[cp.decomp_mapping[0]] ||= {}
@ucd.composition_map[cp.decomp_mapping[0]][cp.decomp_mapping[1]] = cp.code
end
end
end
def normalize_boundary_map
@ucd.boundary.each do |k,v|
if [:lf, :cr].include? k
@ucd.boundary[k] = v[0]
end
end
end
def parse
SOURCES.each do |type, url|
filename = File.join(Dir.tmpdir, "#{url.split('/').last}")
unless File.exist?(filename)
$stderr.puts "Downloading #{url.split('/').last}"
File.open(filename, 'wb') do |target|
open(url) do |source|
source.each_line { |line| target.write line }
end
end
end
File.open(filename) do |file|
file.each_line { |line| send "parse_#{type}".intern, line }
end
end
create_composition_map
normalize_boundary_map
end
def dump_to(filename)
File.open(filename, 'wb') do |f|
f.write Marshal.dump([@ucd.codepoints, @ucd.composition_exclusion, @ucd.composition_map, @ucd.boundary, @ucd.cp1252])
end
end
end
end
if __FILE__ == $0
filename = ActiveSupport::Multibyte::Handlers::UnicodeDatabase.filename
generator = ActiveSupport::Multibyte::Handlers::UnicodeTableGenerator.new
generator.parse
print "Writing to: #{filename}"
generator.dump_to filename
puts " (#{File.size(filename)} bytes)"
end

9
activesupport/lib/active_support/multibyte/handlers/passthru_handler.rb
View File

@@ -1,9 +0,0 @@
# Chars uses this handler when $KCODE is not set to 'UTF8'. Because this handler doesn't define any methods all call
# will be forwarded to String.
class ActiveSupport::Multibyte::Handlers::PassthruHandler #:nodoc:
# Return the original byteoffset
def self.translate_offset(string, byte_offset) #:nodoc:
byte_offset
end
end

564
activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb
View File

@@ -1,564 +0,0 @@
# Contains all the handlers and helper classes
module ActiveSupport::Multibyte::Handlers #:nodoc:
class EncodingError < ArgumentError #:nodoc:
end
class Codepoint #:nodoc:
attr_accessor :code, :combining_class, :decomp_type, :decomp_mapping, :uppercase_mapping, :lowercase_mapping
end
class UnicodeDatabase #:nodoc:
attr_writer :codepoints, :composition_exclusion, :composition_map, :boundary, :cp1252
# self-expiring methods that lazily load the Unicode database and then return the value.
[:codepoints, :composition_exclusion, :composition_map, :boundary, :cp1252].each do |attr_name|
class_eval(<<-EOS, __FILE__, __LINE__)
def #{attr_name}
load
@#{attr_name}
end
EOS
end
# Shortcut to ucd.codepoints[]
def [](index); codepoints[index]; end
# Returns the directory in which the data files are stored
def self.dirname
File.dirname(__FILE__) + '/../../values/'
end
# Returns the filename for the data file for this version
def self.filename
File.expand_path File.join(dirname, "unicode_tables.dat")
end
# Loads the unicode database and returns all the internal objects of UnicodeDatabase
# Once the values have been loaded, define attr_reader methods for the instance variables.
def load
begin
@codepoints, @composition_exclusion, @composition_map, @boundary, @cp1252 = File.open(self.class.filename, 'rb') { |f| Marshal.load f.read }
rescue Exception => e
raise IOError.new("Couldn't load the unicode tables for UTF8Handler (#{e.message}), handler is unusable")
end
@codepoints ||= Hash.new(Codepoint.new)
@composition_exclusion ||= []
@composition_map ||= {}
@boundary ||= {}
@cp1252 ||= {}
# Redefine the === method so we can write shorter rules for grapheme cluster breaks
@boundary.each do |k,_|
@boundary[k].instance_eval do
def ===(other)
detect { |i| i === other } ? true : false
end
end if @boundary[k].kind_of?(Array)
end
# define attr_reader methods for the instance variables
class << self
attr_reader :codepoints, :composition_exclusion, :composition_map, :boundary, :cp1252
end
end
end
# UTF8Handler implements Unicode aware operations for strings, these operations will be used by the Chars
# proxy when $KCODE is set to 'UTF8'.
class UTF8Handler
# Hangul character boundaries and properties
HANGUL_SBASE = 0xAC00
HANGUL_LBASE = 0x1100
HANGUL_VBASE = 0x1161
HANGUL_TBASE = 0x11A7
HANGUL_LCOUNT = 19
HANGUL_VCOUNT = 21
HANGUL_TCOUNT = 28
HANGUL_NCOUNT = HANGUL_VCOUNT * HANGUL_TCOUNT
HANGUL_SCOUNT = 11172
HANGUL_SLAST = HANGUL_SBASE + HANGUL_SCOUNT
HANGUL_JAMO_FIRST = 0x1100
HANGUL_JAMO_LAST = 0x11FF
# All the unicode whitespace
UNICODE_WHITESPACE = [
(0x0009..0x000D).to_a, # White_Space # Cc [5] <control-0009>..<control-000D>
0x0020, # White_Space # Zs SPACE
0x0085, # White_Space # Cc <control-0085>
0x00A0, # White_Space # Zs NO-BREAK SPACE
0x1680, # White_Space # Zs OGHAM SPACE MARK
0x180E, # White_Space # Zs MONGOLIAN VOWEL SEPARATOR
(0x2000..0x200A).to_a, # White_Space # Zs [11] EN QUAD..HAIR SPACE
0x2028, # White_Space # Zl LINE SEPARATOR
0x2029, # White_Space # Zp PARAGRAPH SEPARATOR
0x202F, # White_Space # Zs NARROW NO-BREAK SPACE
0x205F, # White_Space # Zs MEDIUM MATHEMATICAL SPACE
0x3000, # White_Space # Zs IDEOGRAPHIC SPACE
].flatten.freeze
# BOM (byte order mark) can also be seen as whitespace, it's a non-rendering character used to distinguish
# between little and big endian. This is not an issue in utf-8, so it must be ignored.
UNICODE_LEADERS_AND_TRAILERS = UNICODE_WHITESPACE + [65279] # ZERO-WIDTH NO-BREAK SPACE aka BOM
# Borrowed from the Kconv library by Shinji KONO - (also as seen on the W3C site)
UTF8_PAT = /\A(?:
[\x00-\x7f] |
[\xc2-\xdf] [\x80-\xbf] |
\xe0 [\xa0-\xbf] [\x80-\xbf] |
[\xe1-\xef] [\x80-\xbf] [\x80-\xbf] |
\xf0 [\x90-\xbf] [\x80-\xbf] [\x80-\xbf] |
[\xf1-\xf3] [\x80-\xbf] [\x80-\xbf] [\x80-\xbf] |
\xf4 [\x80-\x8f] [\x80-\xbf] [\x80-\xbf]
)*\z/xn
# Returns a regular expression pattern that matches the passed Unicode codepoints
def self.codepoints_to_pattern(array_of_codepoints) #:nodoc:
array_of_codepoints.collect{ |e| [e].pack 'U*' }.join('|')
end
UNICODE_TRAILERS_PAT = /(#{codepoints_to_pattern(UNICODE_LEADERS_AND_TRAILERS)})+\Z/
UNICODE_LEADERS_PAT = /\A(#{codepoints_to_pattern(UNICODE_LEADERS_AND_TRAILERS)})+/
class << self
# ///
# /// BEGIN String method overrides
# ///
# Inserts the passed string at specified codepoint offsets
def insert(str, offset, fragment)
str.replace(
u_unpack(str).insert(
offset,
u_unpack(fragment)
).flatten.pack('U*')
)
end
# Returns the position of the passed argument in the string, counting in codepoints
def index(str, *args)
bidx = str.index(*args)
bidx ? (u_unpack(str.slice(0...bidx)).size) : nil
end
# Works just like the indexed replace method on string, except instead of byte offsets you specify
# character offsets.
#
# Example:
#
# s = "Müller"
# s.chars[2] = "e" # Replace character with offset 2
# s # => "Müeler"
#
# s = "Müller"
# s.chars[1, 2] = "ö" # Replace 2 characters at character offset 1
# s # => "Möler"
def []=(str, *args)
replace_by = args.pop
# Indexed replace with regular expressions already works
return str[*args] = replace_by if args.first.is_a?(Regexp)
result = u_unpack(str)
if args[0].is_a?(Fixnum)
raise IndexError, "index #{args[0]} out of string" if args[0] >= result.length
min = args[0]
max = args[1].nil? ? min : (min + args[1] - 1)
range = Range.new(min, max)
replace_by = [replace_by].pack('U') if replace_by.is_a?(Fixnum)
elsif args.first.is_a?(Range)
raise RangeError, "#{args[0]} out of range" if args[0].min >= result.length
range = args[0]
else
needle = args[0].to_s
min = index(str, needle)
max = min + length(needle) - 1
range = Range.new(min, max)
end
result[range] = u_unpack(replace_by)
str.replace(result.pack('U*'))
end
# Works just like String#rjust, only integer specifies characters instead of bytes.
#
# Example:
#
# "¾ cup".chars.rjust(8).to_s
# # => " ¾ cup"
#
# "¾ cup".chars.rjust(8, " ").to_s # Use non-breaking whitespace
# # => "   ¾ cup"
def rjust(str, integer, padstr=' ')
justify(str, integer, :right, padstr)
end
# Works just like String#ljust, only integer specifies characters instead of bytes.
#
# Example:
#
# "¾ cup".chars.rjust(8).to_s
# # => "¾ cup "
#
# "¾ cup".chars.rjust(8, " ").to_s # Use non-breaking whitespace
# # => "¾ cup   "
def ljust(str, integer, padstr=' ')
justify(str, integer, :left, padstr)
end
# Works just like String#center, only integer specifies characters instead of bytes.
#
# Example:
#
# "¾ cup".chars.center(8).to_s
# # => " ¾ cup "
#
# "¾ cup".chars.center(8, " ").to_s # Use non-breaking whitespace
# # => " ¾ cup  "
def center(str, integer, padstr=' ')
justify(str, integer, :center, padstr)
end
# Does Unicode-aware rstrip
def rstrip(str)
str.gsub(UNICODE_TRAILERS_PAT, '')
end
# Does Unicode-aware lstrip
def lstrip(str)
str.gsub(UNICODE_LEADERS_PAT, '')
end
# Removed leading and trailing whitespace
def strip(str)
str.gsub(UNICODE_LEADERS_PAT, '').gsub(UNICODE_TRAILERS_PAT, '')
end
# Returns the number of codepoints in the string
def size(str)
u_unpack(str).size
end
alias_method :length, :size
# Reverses codepoints in the string.
def reverse(str)
u_unpack(str).reverse.pack('U*')
end
# Implements Unicode-aware slice with codepoints. Slicing on one point returns the codepoints for that
# character.
def slice(str, *args)
if args.size > 2
raise ArgumentError, "wrong number of arguments (#{args.size} for 1)" # Do as if we were native
elsif (args.size == 2 && !(args.first.is_a?(Numeric) || args.first.is_a?(Regexp)))
raise TypeError, "cannot convert #{args.first.class} into Integer" # Do as if we were native
elsif (args.size == 2 && !args[1].is_a?(Numeric))
raise TypeError, "cannot convert #{args[1].class} into Integer" # Do as if we were native
elsif args[0].kind_of? Range
cps = u_unpack(str).slice(*args)
cps.nil? ? nil : cps.pack('U*')
elsif args[0].kind_of? Regexp
str.slice(*args)
elsif args.size == 1 && args[0].kind_of?(Numeric)
u_unpack(str)[args[0]]
else
u_unpack(str).slice(*args).pack('U*')
end
end
alias_method :[], :slice
# Convert characters in the string to uppercase
def upcase(str); to_case :uppercase_mapping, str; end
# Convert characters in the string to lowercase
def downcase(str); to_case :lowercase_mapping, str; end
# Returns a copy of +str+ with the first character converted to uppercase and the remainder to lowercase
def capitalize(str)
upcase(slice(str, 0..0)) + downcase(slice(str, 1..-1) || '')
end
# ///
# /// Extra String methods for unicode operations
# ///
# Returns the KC normalization of the string by default. NFKC is considered the best normalization form for
# passing strings to databases and validations.
#
# * <tt>str</tt> - The string to perform normalization on.
# * <tt>form</tt> - The form you want to normalize in. Should be one of the following:
# <tt>:c</tt>, <tt>:kc</tt>, <tt>:d</tt>, or <tt>:kd</tt>. Default is
# ActiveSupport::Multibyte::DEFAULT_NORMALIZATION_FORM.
def normalize(str, form=ActiveSupport::Multibyte::DEFAULT_NORMALIZATION_FORM)
# See http://www.unicode.org/reports/tr15, Table 1
codepoints = u_unpack(str)
case form
when :d
reorder_characters(decompose_codepoints(:canonical, codepoints))
when :c
compose_codepoints reorder_characters(decompose_codepoints(:canonical, codepoints))
when :kd
reorder_characters(decompose_codepoints(:compatability, codepoints))
when :kc
compose_codepoints reorder_characters(decompose_codepoints(:compatability, codepoints))
else
raise ArgumentError, "#{form} is not a valid normalization variant", caller
end.pack('U*')
end
# Perform decomposition on the characters in the string
def decompose(str)
decompose_codepoints(:canonical, u_unpack(str)).pack('U*')
end
# Perform composition on the characters in the string
def compose(str)
compose_codepoints u_unpack(str).pack('U*')
end
# ///
# /// BEGIN Helper methods for unicode operation
# ///
# Used to translate an offset from bytes to characters, for instance one received from a regular expression match
def translate_offset(str, byte_offset)
return nil if byte_offset.nil?
return 0 if str == ''
chunk = str[0..byte_offset]
begin
begin
chunk.unpack('U*').length - 1
rescue ArgumentError => e
chunk = str[0..(byte_offset+=1)]
# Stop retrying at the end of the string
raise e unless byte_offset < chunk.length
# We damaged a character, retry
retry
end
# Catch the ArgumentError so we can throw our own
rescue ArgumentError
raise EncodingError.new('malformed UTF-8 character')
end
end
# Checks if the string is valid UTF8.
def consumes?(str)
# Unpack is a little bit faster than regular expressions
begin
str.unpack('U*')
true
rescue ArgumentError
false
end
end
# Returns the number of grapheme clusters in the string. This method is very likely to be moved or renamed
# in future versions.
def g_length(str)
g_unpack(str).length
end
# Replaces all the non-utf-8 bytes by their iso-8859-1 or cp1252 equivalent resulting in a valid utf-8 string
def tidy_bytes(str)
str.split(//u).map do |c|
if !UTF8_PAT.match(c)
n = c.unpack('C')[0]
n < 128 ? n.chr :
n < 160 ? [UCD.cp1252[n] || n].pack('U') :
n < 192 ? "\xC2" + n.chr : "\xC3" + (n-64).chr
else
c
end
end.join
end
protected
# Detect whether the codepoint is in a certain character class. Primarily used by the
# grapheme cluster support.
def in_char_class?(codepoint, classes)
classes.detect { |c| UCD.boundary[c] === codepoint } ? true : false
end
# Unpack the string at codepoints boundaries
def u_unpack(str)
begin
str.unpack 'U*'
rescue ArgumentError
raise EncodingError.new('malformed UTF-8 character')
end
end
# Unpack the string at grapheme boundaries instead of codepoint boundaries
def g_unpack(str)
codepoints = u_unpack(str)
unpacked = []
pos = 0
marker = 0
eoc = codepoints.length
while(pos < eoc)
pos += 1
previous = codepoints[pos-1]
current = codepoints[pos]
if (
# CR X LF
one = ( previous == UCD.boundary[:cr] and current == UCD.boundary[:lf] ) or
# L X (L|V|LV|LVT)
two = ( UCD.boundary[:l] === previous and in_char_class?(current, [:l,:v,:lv,:lvt]) ) or
# (LV|V) X (V|T)
three = ( in_char_class?(previous, [:lv,:v]) and in_char_class?(current, [:v,:t]) ) or
# (LVT|T) X (T)
four = ( in_char_class?(previous, [:lvt,:t]) and UCD.boundary[:t] === current ) or
# X Extend
five = (UCD.boundary[:extend] === current)
)
else
unpacked << codepoints[marker..pos-1]
marker = pos
end
end
unpacked
end
# Reverse operation of g_unpack
def g_pack(unpacked)
unpacked.flatten
end
# Justifies a string in a certain way. Valid values for <tt>way</tt> are <tt>:right</tt>, <tt>:left</tt> and
# <tt>:center</tt>. Is primarily used as a helper method by <tt>rjust</tt>, <tt>ljust</tt> and <tt>center</tt>.
def justify(str, integer, way, padstr=' ')
raise ArgumentError, "zero width padding" if padstr.length == 0
padsize = integer - size(str)
padsize = padsize > 0 ? padsize : 0
case way
when :right
str.dup.insert(0, padding(padsize, padstr))
when :left
str.dup.insert(-1, padding(padsize, padstr))
when :center
lpad = padding((padsize / 2.0).floor, padstr)
rpad = padding((padsize / 2.0).ceil, padstr)
str.dup.insert(0, lpad).insert(-1, rpad)
end
end
# Generates a padding string of a certain size.
def padding(padsize, padstr=' ')
if padsize != 0
slice(padstr * ((padsize / size(padstr)) + 1), 0, padsize)
else
''
end
end
# Convert characters to a different case
def to_case(way, str)
u_unpack(str).map do |codepoint|
cp = UCD[codepoint]
unless cp.nil?
ncp = cp.send(way)
ncp > 0 ? ncp : codepoint
else
codepoint
end
end.pack('U*')
end
# Re-order codepoints so the string becomes canonical
def reorder_characters(codepoints)
length = codepoints.length- 1
pos = 0
while pos < length do
cp1, cp2 = UCD[codepoints[pos]], UCD[codepoints[pos+1]]
if (cp1.combining_class > cp2.combining_class) && (cp2.combining_class > 0)
codepoints[pos..pos+1] = cp2.code, cp1.code
pos += (pos > 0 ? -1 : 1)
else
pos += 1
end
end
codepoints
end
# Decompose composed characters to the decomposed form
def decompose_codepoints(type, codepoints)
codepoints.inject([]) do |decomposed, cp|
# if it's a hangul syllable starter character
if HANGUL_SBASE <= cp and cp < HANGUL_SLAST
sindex = cp - HANGUL_SBASE
ncp = [] # new codepoints
ncp << HANGUL_LBASE + sindex / HANGUL_NCOUNT
ncp << HANGUL_VBASE + (sindex % HANGUL_NCOUNT) / HANGUL_TCOUNT
tindex = sindex % HANGUL_TCOUNT
ncp << (HANGUL_TBASE + tindex) unless tindex == 0
decomposed.concat ncp
# if the codepoint is decomposable in with the current decomposition type
elsif (ncp = UCD[cp].decomp_mapping) and (!UCD[cp].decomp_type || type == :compatability)
decomposed.concat decompose_codepoints(type, ncp.dup)
else
decomposed << cp
end
end
end
# Compose decomposed characters to the composed form
def compose_codepoints(codepoints)
pos = 0
eoa = codepoints.length - 1
starter_pos = 0
starter_char = codepoints[0]
previous_combining_class = -1
while pos < eoa
pos += 1
lindex = starter_char - HANGUL_LBASE
# -- Hangul
if 0 <= lindex and lindex < HANGUL_LCOUNT
vindex = codepoints[starter_pos+1] - HANGUL_VBASE rescue vindex = -1
if 0 <= vindex and vindex < HANGUL_VCOUNT
tindex = codepoints[starter_pos+2] - HANGUL_TBASE rescue tindex = -1
if 0 <= tindex and tindex < HANGUL_TCOUNT
j = starter_pos + 2
eoa -= 2
else
tindex = 0
j = starter_pos + 1
eoa -= 1
end
codepoints[starter_pos..j] = (lindex * HANGUL_VCOUNT + vindex) * HANGUL_TCOUNT + tindex + HANGUL_SBASE
end
starter_pos += 1
starter_char = codepoints[starter_pos]
# -- Other characters
else
current_char = codepoints[pos]
current = UCD[current_char]
if current.combining_class > previous_combining_class
if ref = UCD.composition_map[starter_char]
composition = ref[current_char]
else
composition = nil
end
unless composition.nil?
codepoints[starter_pos] = composition
starter_char = composition
codepoints.delete_at pos
eoa -= 1
pos -= 1
previous_combining_class = -1
else
previous_combining_class = current.combining_class
end
else
previous_combining_class = current.combining_class
end
if current.combining_class == 0
starter_pos = pos
starter_char = codepoints[pos]
end
end
end
codepoints
end
# UniCode Database
UCD = UnicodeDatabase.new
end
end
end

43
activesupport/lib/active_support/multibyte/handlers/utf8_handler_proc.rb
View File

@@ -1,43 +0,0 @@
# Methods in this handler call functions in the utf8proc ruby extension. These are significantly faster than the
# pure ruby versions. Chars automatically uses this handler when it can load the utf8proc extension. For
# documentation on handler methods see UTF8Handler.
class ActiveSupport::Multibyte::Handlers::UTF8HandlerProc < ActiveSupport::Multibyte::Handlers::UTF8Handler #:nodoc:
class << self
def normalize(str, form=ActiveSupport::Multibyte::DEFAULT_NORMALIZATION_FORM) #:nodoc:
codepoints = str.unpack('U*')
case form
when :d
utf8map(str, :stable)
when :c
utf8map(str, :stable, :compose)
when :kd
utf8map(str, :stable, :compat)
when :kc
utf8map(str, :stable, :compose, :compat)
else
raise ArgumentError, "#{form} is not a valid normalization variant", caller
end
end
def decompose(str) #:nodoc:
utf8map(str, :stable)
end
def downcase(str) #:nodoc:c
utf8map(str, :casefold)
end
protected
def utf8map(str, *option_array) #:nodoc:
options = 0
option_array.each do |option|
flag = Utf8Proc::Options[option]
raise ArgumentError, "Unknown argument given to utf8map." unless
flag
options |= flag
end
return Utf8Proc::utf8map(str, options)
end
end
end

71
activesupport/lib/active_support/multibyte/unicode_database.rb Normal file
View File

@@ -0,0 +1,71 @@
# encoding: utf-8
module ActiveSupport #:nodoc:
module Multibyte #:nodoc:
# Holds data about a codepoint in the Unicode database
class Codepoint
attr_accessor :code, :combining_class, :decomp_type, :decomp_mapping, :uppercase_mapping, :lowercase_mapping
end
# Holds static data from the Unicode database
class UnicodeDatabase
ATTRIBUTES = :codepoints, :composition_exclusion, :composition_map, :boundary, :cp1252
attr_writer(*ATTRIBUTES)
def initialize
@codepoints = Hash.new(Codepoint.new)
@composition_exclusion = []
@composition_map = {}
@boundary = {}
@cp1252 = {}
end
# Lazy load the Unicode database so it's only loaded when it's actually used
ATTRIBUTES.each do |attr_name|
class_eval(<<-EOS, __FILE__, __LINE__)
def #{attr_name}
load
@#{attr_name}
end
EOS
end
# Loads the Unicode database and returns all the internal objects of UnicodeDatabase.
def load
begin
@codepoints, @composition_exclusion, @composition_map, @boundary, @cp1252 = File.open(self.class.filename, 'rb') { |f| Marshal.load f.read }
rescue Exception => e
raise IOError.new("Couldn't load the Unicode tables for UTF8Handler (#{e.message}), ActiveSupport::Multibyte is unusable")
end
# Redefine the === method so we can write shorter rules for grapheme cluster breaks
@boundary.each do |k,_|
@boundary[k].instance_eval do
def ===(other)
detect { |i| i === other } ? true : false
end
end if @boundary[k].kind_of?(Array)
end
# define attr_reader methods for the instance variables
class << self
attr_reader(*ATTRIBUTES)
end
end
# Returns the directory in which the data files are stored
def self.dirname
File.dirname(__FILE__) + '/../values/'
end
# Returns the filename for the data file for this version
def self.filename
File.expand_path File.join(dirname, "unicode_tables.dat")
end
end
# UniCode Database
UCD = UnicodeDatabase.new
end
end

BIN
activesupport/lib/active_support/values/unicode_tables.dat
View File

Binary file not shown.

8
activesupport/test/abstract_unit.rb
View File

@@ -1,9 +1,15 @@
# encoding: utf-8
require 'test/unit'
$:.unshift "#{File.dirname(__FILE__)}/../lib"
$:.unshift File.dirname(__FILE__)
require 'active_support'
if RUBY_VERSION < '1.9'
$KCODE = 'UTF8'
end
def uses_gem(gem_name, test_name, version = '> 0')
require 'rubygems'
gem gem_name.to_s, version
@@ -21,4 +27,4 @@ unless defined? uses_mocha
end
# Show backtraces for deprecated behavior for quicker cleanup.
ActiveSupport::Deprecation.debug = true
ActiveSupport::Deprecation.debug = true

650
activesupport/test/multibyte_chars_test.rb
View File

@@ -1,186 +1,528 @@
# encoding: utf-8
require 'abstract_unit'
if RUBY_VERSION < '1.9'
module MultibyteTest
UNICODE_STRING = 'こにちわ'
ASCII_STRING = 'ohayo'
BYTE_STRING = "\270\236\010\210\245"
$KCODE = 'UTF8'
def chars(str)
ActiveSupport::Multibyte::Chars.new(str)
end
def inspect_codepoints(str)
str.to_s.unpack("U*").map{|cp| cp.to_s(16) }.join(' ')
end
def assert_equal_codepoints(expected, actual, message=nil)
assert_equal(inspect_codepoints(expected), inspect_codepoints(actual), message)
end
end
class String
def __string_for_multibyte_testing; self; end
def __string_for_multibyte_testing!; self; end
end
class MultibyteCharsTest < Test::Unit::TestCase
include MultibyteTest
class CharsTest < Test::Unit::TestCase
def setup
@s = {
:utf8 => "Abcd Блå ffi блa 埋",
:ascii => "asci ias c iia s",
:bytes => "\270\236\010\210\245"
}
@chars = ActiveSupport::Multibyte::Chars.new UNICODE_STRING
end
def test_sanity
@s.each do |t, s|
assert s.respond_to?(:chars), "All string should have the chars method (#{t})"
assert s.respond_to?(:to_s), "All string should have the to_s method (#{t})"
assert_kind_of ActiveSupport::Multibyte::Chars, s.chars, "#chars should return an instance of Chars (#{t})"
end
def test_wraps_the_original_string
assert_equal UNICODE_STRING, @chars.to_s
assert_equal UNICODE_STRING, @chars.wrapped_string
end
def test_comparability
@s.each do |t, s|
assert_equal s, s.chars.to_s, "Chars#to_s should return enclosed string unchanged"
end
def test_should_allow_method_calls_to_string
assert_nothing_raised do
assert_equal "a", "a", "Normal string comparisons should be unaffected"
assert_not_equal "a", "b", "Normal string comparisons should be unaffected"
assert_not_equal "a".chars, "b".chars, "Chars objects should be comparable"
assert_equal "a".chars, "A".downcase.chars, "Chars objects should be comparable to each other"
assert_equal "a".chars, "A".downcase, "Chars objects should be comparable to strings coming from elsewhere"
@chars.__string_for_multibyte_testing
end
assert !@s[:utf8].eql?(@s[:utf8].chars), "Strict comparison is not supported"
assert_equal @s[:utf8], @s[:utf8].chars, "Chars should be compared by their enclosed string"
other_string = @s[:utf8].dup
assert_equal other_string, @s[:utf8].chars, "Chars should be compared by their enclosed string"
assert_equal other_string.chars, @s[:utf8].chars, "Chars should be compared by their enclosed string"
strings = ['builder'.chars, 'armor'.chars, 'zebra'.chars]
strings.sort!
assert_equal ['armor', 'builder', 'zebra'], strings, "Chars should be sortable based on their enclosed string"
# This leads to a StackLevelTooDeep exception if the comparison is not wired properly
assert_raise(NameError) do
Chars
end
end
def test_utf8?
assert @s[:utf8].is_utf8?, "UTF-8 strings are UTF-8"
assert @s[:ascii].is_utf8?, "All ASCII strings are also valid UTF-8"
assert !@s[:bytes].is_utf8?, "This bytestring isn't UTF-8"
end
# The test for the following methods are defined here because they can only be defined on the Chars class for
# various reasons
def test_gsub
assert_equal 'éxa', 'éda'.chars.gsub(/d/, 'x')
with_kcode('none') do
assert_equal 'éxa', 'éda'.chars.gsub(/d/, 'x')
end
end
def test_split
word = "efficient"
chars = ["e", "ffi", "c", "i", "e", "n", "t"]
assert_equal chars, word.split(//)
assert_equal chars, word.chars.split(//)
assert_kind_of ActiveSupport::Multibyte::Chars, word.chars.split(//).first, "Split should return Chars instances"
end
def test_regexp
with_kcode('none') do
assert_equal 12, (@s[:utf8].chars =~ /ffi/),
"Regex matching should be bypassed to String"
end
with_kcode('UTF8') do
assert_equal 9, (@s[:utf8].chars =~ /ffi/),
"Regex matching should be unicode aware"
assert_nil((''.chars =~ /\d+/),
"Non-matching regular expressions should return nil")
assert_raises NoMethodError do
@chars.__unknown_method
end
end
def test_pragma
if RUBY_VERSION < '1.9'
with_kcode('UTF8') do
assert " ".chars.send(:utf8_pragma?), "UTF8 pragma should be on because KCODE is UTF8"
def test_forwarded_method_calls_should_return_new_chars_instance
assert @chars.__string_for_multibyte_testing.kind_of?(ActiveSupport::Multibyte::Chars)
assert_not_equal @chars.object_id, @chars.__string_for_multibyte_testing.object_id
end
def test_forwarded_bang_method_calls_should_return_the_original_chars_instance
assert @chars.__string_for_multibyte_testing!.kind_of?(ActiveSupport::Multibyte::Chars)
assert_equal @chars.object_id, @chars.__string_for_multibyte_testing!.object_id
end
def test_methods_are_forwarded_to_wrapped_string_for_byte_strings
assert_equal BYTE_STRING.class, BYTE_STRING.mb_chars.class
end
def test_should_concatenate
assert_equal 'ab', 'a'.mb_chars + 'b'
assert_equal 'ab', 'a' + 'b'.mb_chars
assert_equal 'ab', 'a'.mb_chars + 'b'.mb_chars
assert_equal 'ab', 'a'.mb_chars << 'b'
assert_equal 'ab', 'a' << 'b'.mb_chars
assert_equal 'ab', 'a'.mb_chars << 'b'.mb_chars
end
if RUBY_VERSION < '1.9'
def test_concatenation_should_return_a_proxy_class_instance
assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars + 'b').class
assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars << 'b').class
end
def test_ascii_strings_are_treated_at_utf8_strings
assert_equal ActiveSupport::Multibyte.proxy_class, ASCII_STRING.mb_chars.class
end
def test_concatenate_should_return_proxy_instance
assert(('a'.mb_chars + 'b').kind_of?(ActiveSupport::Multibyte::Chars))
assert(('a'.mb_chars + 'b'.mb_chars).kind_of?(ActiveSupport::Multibyte::Chars))
assert(('a'.mb_chars << 'b').kind_of?(ActiveSupport::Multibyte::Chars))
assert(('a'.mb_chars << 'b'.mb_chars).kind_of?(ActiveSupport::Multibyte::Chars))
end
end
end
class MultibyteCharsUTF8BehaviourTest < Test::Unit::TestCase
include MultibyteTest
def setup
@chars = UNICODE_STRING.dup.mb_chars
# NEWLINE, SPACE, EM SPACE
@whitespace = "\n#{[32, 8195].pack('U*')}"
@whitespace.force_encoding(Encoding::UTF_8) if @whitespace.respond_to?(:force_encoding)
@byte_order_mark = [65279].pack('U')
end
if RUBY_VERSION < '1.9'
def test_split_should_return_an_array_of_chars_instances
@chars.split(//).each do |character|
assert character.kind_of?(ActiveSupport::Multibyte::Chars)
end
with_kcode('none') do
assert !" ".chars.send(:utf8_pragma?), "UTF8 pragma should be off because KCODE is not UTF8"
end
def test_indexed_insert_accepts_fixnums
@chars[2] = 32
assert_equal 'こに わ', @chars
end
def test_overridden_bang_methods_return_self
[:rstrip!, :lstrip!, :strip!, :reverse!, :upcase!, :downcase!, :capitalize!].each do |method|
assert_equal @chars.object_id, @chars.send(method).object_id
end
else
assert !" ".chars.send(:utf8_pragma?), "UTF8 pragma should be off in Ruby 1.9"
assert_equal @chars.object_id, @chars.slice!(1).object_id
end
def test_overridden_bang_methods_change_wrapped_string
[:rstrip!, :lstrip!, :strip!, :reverse!, :upcase!, :downcase!].each do |method|
original = ' Café '
proxy = chars(original.dup)
proxy.send(method)
assert_not_equal original, proxy.to_s
end
proxy = chars('Café')
proxy.slice!(3)
assert_equal 'é', proxy.to_s
proxy = chars('òu')
proxy.capitalize!
assert_equal 'Òu', proxy.to_s
end
end
def test_handler_setting
handler = ''.chars.handler
ActiveSupport::Multibyte::Chars.handler = :first
assert_equal :first, ''.chars.handler
ActiveSupport::Multibyte::Chars.handler = :second
assert_equal :second, ''.chars.handler
assert_raise(NoMethodError) do
''.chars.handler.split
end
ActiveSupport::Multibyte::Chars.handler = handler
end
def test_method_chaining
assert_kind_of ActiveSupport::Multibyte::Chars, ''.chars.downcase
assert_kind_of ActiveSupport::Multibyte::Chars, ''.chars.strip, "Strip should return a Chars object"
assert_kind_of ActiveSupport::Multibyte::Chars, ''.chars.downcase.strip, "The Chars object should be " +
"forwarded down the call path for chaining"
assert_equal 'foo', " FOO ".chars.normalize.downcase.strip, "The Chars that results from the " +
" operations should be comparable to the string value of the result"
end
def test_passthrough_on_kcode
# The easiest way to check if the passthrough is in place is through #size
with_kcode('none') do
assert_equal 26, @s[:utf8].chars.size
end
with_kcode('UTF8') do
assert_equal 17, @s[:utf8].chars.size
if RUBY_VERSION >= '1.9'
def test_unicode_string_should_have_utf8_encoding
assert_equal Encoding::UTF_8, UNICODE_STRING.encoding
end
end
def test_destructiveness
# Note that we're testing the destructiveness here and not the correct behaviour of the methods
str = 'ac'
str.chars.insert(1, 'b')
assert_equal 'abc', str, 'Insert should be destructive for a string'
str = 'ac'
str.chars.reverse!
assert_equal 'ca', str, 'reverse! should be destructive for a string'
def test_should_be_equal_to_the_wrapped_string
assert_equal UNICODE_STRING, @chars
assert_equal @chars, UNICODE_STRING
end
def test_resilience
assert_nothing_raised do
assert_equal 5, @s[:bytes].chars.size, "The sequence contains five interpretable bytes"
end
reversed = [0xb8, 0x17e, 0x8, 0x2c6, 0xa5].reverse.pack('U*')
assert_nothing_raised do
assert_equal reversed, @s[:bytes].chars.reverse.to_s, "Reversing the string should only yield interpretable bytes"
end
assert_nothing_raised do
@s[:bytes].chars.reverse!
assert_equal reversed, @s[:bytes].to_s, "Reversing the string should only yield interpretable bytes"
def test_should_not_be_equal_to_an_other_string
assert_not_equal @chars, 'other'
assert_not_equal 'other', @chars
end
def test_should_return_character_offset_for_regexp_matches
assert_nil(@chars =~ /wrong/u)
assert_equal 0, (@chars =~ /こ/u)
assert_equal 1, (@chars =~ /に/u)
assert_equal 3, (@chars =~ /わ/u)
end
def test_should_use_character_offsets_for_insert_offsets
assert_equal '', ''.mb_chars.insert(0, '')
assert_equal 'こわにちわ', @chars.insert(1, 'わ')
assert_equal 'こわわわにちわ', @chars.insert(2, 'わわ')
assert_equal 'わこわわわにちわ', @chars.insert(0, 'わ')
assert_equal 'わこわわわにちわ', @chars.wrapped_string if RUBY_VERSION < '1.9'
end
def test_insert_should_be_destructive
@chars.insert(1, 'わ')
assert_equal 'こわにちわ', @chars
end
def test_insert_throws_index_error
assert_raises(IndexError) { @chars.insert(12, 'わ') }
end
def test_should_know_if_one_includes_the_other
assert @chars.include?('')
assert @chars.include?('ち')
assert @chars.include?('わ')
assert !@chars.include?('こちわ')
assert !@chars.include?('a')
end
def test_include_raises_type_error_when_nil_is_passed
assert_raises(TypeError) do
@chars.include?(nil)
end
end
def test_duck_typing
assert_equal true, 'test'.chars.respond_to?(:strip)
assert_equal true, 'test'.chars.respond_to?(:normalize)
assert_equal true, 'test'.chars.respond_to?(:normalize!)
assert_equal false, 'test'.chars.respond_to?(:a_method_that_doesnt_exist)
def test_index_should_return_character_offset
assert_nil @chars.index('u')
assert_equal 0, @chars.index('こに')
assert_equal 2, @chars.index('ち')
assert_equal 3, @chars.index('わ')
end
def test_indexed_insert_should_take_character_offsets
@chars[2] = 'a'
assert_equal 'こにaわ', @chars
@chars[2] = 'ηη'
assert_equal 'こにηηわ', @chars
@chars[3, 2] = 'λλλ'
assert_equal 'こにηλλλ', @chars
@chars[1, 0] = "λ"
assert_equal 'こλにηλλλ', @chars
@chars[4..6] = "ηη"
assert_equal 'こλにηηη', @chars
@chars[/ηη/] = "λλλ"
assert_equal 'こλにλλλη', @chars
@chars[/(λλ)(.)/, 2] = "α"
assert_equal 'こλにλλαη', @chars
@chars["α"] = "¢"
assert_equal 'こλにλλ¢η', @chars
@chars["λλ"] = "ααα"
assert_equal 'こλにααα¢η', @chars
end
def test_indexed_insert_should_raise_on_index_overflow
before = @chars.to_s
assert_raises(IndexError) { @chars[10] = 'a' }
assert_raises(IndexError) { @chars[10, 4] = 'a' }
assert_raises(IndexError) { @chars[/ii/] = 'a' }
assert_raises(IndexError) { @chars[/()/, 10] = 'a' }
assert_equal before, @chars
end
def test_indexed_insert_should_raise_on_range_overflow
before = @chars.to_s
assert_raises(RangeError) { @chars[10..12] = 'a' }
assert_equal before, @chars
end
def test_rjust_should_raise_argument_errors_on_bad_arguments
assert_raises(ArgumentError) { @chars.rjust(10, '') }
assert_raises(ArgumentError) { @chars.rjust }
end
def test_rjust_should_count_characters_instead_of_bytes
assert_equal UNICODE_STRING, @chars.rjust(-3)
assert_equal UNICODE_STRING, @chars.rjust(0)
assert_equal UNICODE_STRING, @chars.rjust(4)
assert_equal " #{UNICODE_STRING}", @chars.rjust(5)
assert_equal " #{UNICODE_STRING}", @chars.rjust(7)
assert_equal "---#{UNICODE_STRING}", @chars.rjust(7, '-')
assert_equal "ααα#{UNICODE_STRING}", @chars.rjust(7, 'α')
assert_equal "aba#{UNICODE_STRING}", @chars.rjust(7, 'ab')
assert_equal "αηα#{UNICODE_STRING}", @chars.rjust(7, 'αη')
assert_equal "αηαη#{UNICODE_STRING}", @chars.rjust(8, 'αη')
end
def test_ljust_should_raise_argument_errors_on_bad_arguments
assert_raises(ArgumentError) { @chars.ljust(10, '') }
assert_raises(ArgumentError) { @chars.ljust }
end
def test_ljust_should_count_characters_instead_of_bytes
assert_equal UNICODE_STRING, @chars.ljust(-3)
assert_equal UNICODE_STRING, @chars.ljust(0)
assert_equal UNICODE_STRING, @chars.ljust(4)
assert_equal "#{UNICODE_STRING} ", @chars.ljust(5)
assert_equal "#{UNICODE_STRING} ", @chars.ljust(7)
assert_equal "#{UNICODE_STRING}---", @chars.ljust(7, '-')
assert_equal "#{UNICODE_STRING}ααα", @chars.ljust(7, 'α')
assert_equal "#{UNICODE_STRING}aba", @chars.ljust(7, 'ab')
assert_equal "#{UNICODE_STRING}αηα", @chars.ljust(7, 'αη')
assert_equal "#{UNICODE_STRING}αηαη", @chars.ljust(8, 'αη')
end
def test_center_should_raise_argument_errors_on_bad_arguments
assert_raises(ArgumentError) { @chars.center(10, '') }
assert_raises(ArgumentError) { @chars.center }
end
def test_center_should_count_charactes_instead_of_bytes
assert_equal UNICODE_STRING, @chars.center(-3)
assert_equal UNICODE_STRING, @chars.center(0)
assert_equal UNICODE_STRING, @chars.center(4)
assert_equal "#{UNICODE_STRING} ", @chars.center(5)
assert_equal " #{UNICODE_STRING} ", @chars.center(6)
assert_equal " #{UNICODE_STRING} ", @chars.center(7)
assert_equal "--#{UNICODE_STRING}--", @chars.center(8, '-')
assert_equal "--#{UNICODE_STRING}---", @chars.center(9, '-')
assert_equal "αα#{UNICODE_STRING}αα", @chars.center(8, 'α')
assert_equal "αα#{UNICODE_STRING}ααα", @chars.center(9, 'α')
assert_equal "a#{UNICODE_STRING}ab", @chars.center(7, 'ab')
assert_equal "ab#{UNICODE_STRING}ab", @chars.center(8, 'ab')
assert_equal "abab#{UNICODE_STRING}abab", @chars.center(12, 'ab')
assert_equal "α#{UNICODE_STRING}αη", @chars.center(7, 'αη')
assert_equal "αη#{UNICODE_STRING}αη", @chars.center(8, 'αη')
end
def test_lstrip_strips_whitespace_from_the_left_of_the_string
assert_equal UNICODE_STRING, UNICODE_STRING.mb_chars.lstrip
assert_equal UNICODE_STRING, (@whitespace + UNICODE_STRING).mb_chars.lstrip
assert_equal UNICODE_STRING + @whitespace, (@whitespace + UNICODE_STRING + @whitespace).mb_chars.lstrip
end
def test_rstrip_strips_whitespace_from_the_right_of_the_string
assert_equal UNICODE_STRING, UNICODE_STRING.mb_chars.rstrip
assert_equal UNICODE_STRING, (UNICODE_STRING + @whitespace).mb_chars.rstrip
assert_equal @whitespace + UNICODE_STRING, (@whitespace + UNICODE_STRING + @whitespace).mb_chars.rstrip
end
def test_strip_strips_whitespace
assert_equal UNICODE_STRING, UNICODE_STRING.mb_chars.strip
assert_equal UNICODE_STRING, (@whitespace + UNICODE_STRING).mb_chars.strip
assert_equal UNICODE_STRING, (UNICODE_STRING + @whitespace).mb_chars.strip
assert_equal UNICODE_STRING, (@whitespace + UNICODE_STRING + @whitespace).mb_chars.strip
end
def test_stripping_whitespace_leaves_whitespace_within_the_string_intact
string_with_whitespace = UNICODE_STRING + @whitespace + UNICODE_STRING
assert_equal string_with_whitespace, string_with_whitespace.mb_chars.strip
assert_equal string_with_whitespace, string_with_whitespace.mb_chars.lstrip
assert_equal string_with_whitespace, string_with_whitespace.mb_chars.rstrip
end
def test_size_returns_characters_instead_of_bytes
assert_equal 0, ''.mb_chars.size
assert_equal 4, @chars.size
assert_equal 4, @chars.length
assert_equal 5, ASCII_STRING.mb_chars.size
end
def test_reverse_reverses_characters
assert_equal '', ''.mb_chars.reverse
assert_equal 'わちにこ', @chars.reverse
end
def test_slice_should_take_character_offsets
assert_equal nil, ''.mb_chars.slice(0)
assert_equal 'こ', @chars.slice(0)
assert_equal 'わ', @chars.slice(3)
assert_equal nil, ''.mb_chars.slice(-1..1)
assert_equal '', ''.mb_chars.slice(0..10)
assert_equal 'にちわ', @chars.slice(1..3)
assert_equal 'にちわ', @chars.slice(1, 3)
assert_equal 'こ', @chars.slice(0, 1)
assert_equal 'ちわ', @chars.slice(2..10)
assert_equal '', @chars.slice(4..10)
assert_equal 'に', @chars.slice(/に/u)
assert_equal 'にち', @chars.slice(/に\w/u)
assert_equal nil, @chars.slice(/unknown/u)
assert_equal 'にち', @chars.slice(/(にち)/u, 1)
assert_equal nil, @chars.slice(/(にち)/u, 2)
assert_equal nil, @chars.slice(7..6)
end
def test_slice_should_throw_exceptions_on_invalid_arguments
assert_raise(TypeError) { @chars.slice(2..3, 1) }
assert_raise(TypeError) { @chars.slice(1, 2..3) }
assert_raise(ArgumentError) { @chars.slice(1, 1, 1) }
end
def test_upcase_should_upcase_ascii_characters
assert_equal '', ''.mb_chars.upcase
assert_equal 'ABC', 'aBc'.mb_chars.upcase
end
def test_downcase_should_downcase_ascii_characters
assert_equal '', ''.mb_chars.downcase
assert_equal 'abc', 'aBc'.mb_chars.downcase
end
def test_capitalize_should_work_on_ascii_characters
assert_equal '', ''.mb_chars.capitalize
assert_equal 'Abc', 'abc'.mb_chars.capitalize
end
def test_respond_to_knows_which_methods_the_proxy_responds_to
assert ''.mb_chars.respond_to?(:slice) # Defined on Chars
assert ''.mb_chars.respond_to?(:capitalize!) # Defined on Chars
assert ''.mb_chars.respond_to?(:gsub) # Defined on String
assert !''.mb_chars.respond_to?(:undefined_method) # Not defined
end
def test_acts_like_string
assert 'Bambi'.chars.acts_like_string?
assert 'Bambi'.mb_chars.acts_like_string?
end
end
protected
# The default Multibyte Chars proxy has more features than the normal string implementation. Tests
# for the implementation of these features should run on all Ruby versions and shouldn't be tested
# through the proxy methods.
class MultibyteCharsExtrasTest < Test::Unit::TestCase
include MultibyteTest
def with_kcode(kcode)
old_kcode, $KCODE = $KCODE, kcode
begin
yield
ensure
$KCODE = old_kcode
if RUBY_VERSION >= '1.9'
def test_tidy_bytes_is_broken_on_1_9_0
assert_raises(ArgumentError) do
assert_equal_codepoints [0xfffd].pack('U'), chars("\xef\xbf\xbd").tidy_bytes
end
end
end
end
def test_upcase_should_be_unicode_aware
assert_equal "АБВГД\0F", chars("аБвгд\0f").upcase
assert_equal 'こにちわ', chars('こにちわ').upcase
end
def test_downcase_should_be_unicode_aware
assert_equal "абвгд\0f", chars("аБвгд\0f").downcase
assert_equal 'こにちわ', chars('こにちわ').downcase
end
def test_capitalize_should_be_unicode_aware
{ 'аБвг аБвг' => 'Абвг абвг',
'аБвг АБВГ' => 'Абвг абвг',
'АБВГ АБВГ' => 'Абвг абвг',
'' => '' }.each do |f,t|
assert_equal t, chars(f).capitalize
end
end
def test_composition_exclusion_is_set_up_properly
# Normalization of DEVANAGARI LETTER QA breaks when composition exclusion isn't used correctly
qa = [0x915, 0x93c].pack('U*')
assert_equal qa, chars(qa).normalize(:c)
end
# Test for the Public Review Issue #29, bad explanation of composition might lead to a
# bad implementation: http://www.unicode.org/review/pr-29.html
def test_normalization_C_pri_29
[
[0x0B47, 0x0300, 0x0B3E],
[0x1100, 0x0300, 0x1161]
].map { |c| c.pack('U*') }.each do |c|
assert_equal_codepoints c, chars(c).normalize(:c)
end
end
def test_normalization_shouldnt_strip_null_bytes
null_byte_str = "Test\0test"
assert_equal null_byte_str, chars(null_byte_str).normalize(:kc)
assert_equal null_byte_str, chars(null_byte_str).normalize(:c)
assert_equal null_byte_str, chars(null_byte_str).normalize(:d)
assert_equal null_byte_str, chars(null_byte_str).normalize(:kd)
assert_equal null_byte_str, chars(null_byte_str).decompose
assert_equal null_byte_str, chars(null_byte_str).compose
end
def test_simple_normalization
comp_str = [
44, # LATIN CAPITAL LETTER D
307, # COMBINING DOT ABOVE
328, # COMBINING OGONEK
323 # COMBINING DOT BELOW
].pack("U*")
assert_equal_codepoints '', chars('').normalize
assert_equal_codepoints [44,105,106,328,323].pack("U*"), chars(comp_str).normalize(:kc).to_s
assert_equal_codepoints [44,307,328,323].pack("U*"), chars(comp_str).normalize(:c).to_s
assert_equal_codepoints [44,307,110,780,78,769].pack("U*"), chars(comp_str).normalize(:d).to_s
assert_equal_codepoints [44,105,106,110,780,78,769].pack("U*"), chars(comp_str).normalize(:kd).to_s
end
def test_should_compute_grapheme_length
[
['', 0],
['abc', 3],
['こにちわ', 4],
[[0x0924, 0x094D, 0x0930].pack('U*'), 2],
[%w(cr lf), 1],
[%w(l l), 1],
[%w(l v), 1],
[%w(l lv), 1],
[%w(l lvt), 1],
[%w(lv v), 1],
[%w(lv t), 1],
[%w(v v), 1],
[%w(v t), 1],
[%w(lvt t), 1],
[%w(t t), 1],
[%w(n extend), 1],
[%w(n n), 2],
[%w(n cr lf n), 3],
[%w(n l v t), 2]
].each do |input, expected_length|
if input.kind_of?(Array)
str = string_from_classes(input)
else
str = input
end
assert_equal expected_length, chars(str).g_length
end
end
def test_tidy_bytes_should_tidy_bytes
byte_string = "\270\236\010\210\245"
tidy_string = [0xb8, 0x17e, 0x8, 0x2c6, 0xa5].pack('U*')
ascii_padding = 'aa'
utf8_padding = 'éé'
assert_equal_codepoints tidy_string, chars(byte_string).tidy_bytes
assert_equal_codepoints ascii_padding.dup.insert(1, tidy_string),
chars(ascii_padding.dup.insert(1, byte_string)).tidy_bytes
assert_equal_codepoints utf8_padding.dup.insert(2, tidy_string),
chars(utf8_padding.dup.insert(2, byte_string)).tidy_bytes
assert_nothing_raised { chars(byte_string).tidy_bytes.to_s.unpack('U*') }
assert_equal_codepoints "\xC3\xA7", chars("\xE7").tidy_bytes # iso_8859_1: small c cedilla
assert_equal_codepoints "\xE2\x80\x9C", chars("\x93").tidy_bytes # win_1252: left smart quote
assert_equal_codepoints "\xE2\x82\xAC", chars("\x80").tidy_bytes # win_1252: euro
assert_equal_codepoints "\x00", chars("\x00").tidy_bytes # null char
assert_equal_codepoints [0xfffd].pack('U'), chars("\xef\xbf\xbd").tidy_bytes # invalid char
rescue ArgumentError => e
raise e if RUBY_VERSION < '1.9'
end
private
def string_from_classes(classes)
# Characters from the character classes as described in UAX #29
character_from_class = {
:l => 0x1100, :v => 0x1160, :t => 0x11A8, :lv => 0xAC00, :lvt => 0xAC01, :cr => 0x000D, :lf => 0x000A,
:extend => 0x094D, :n => 0x64
}
classes.collect do |k|
character_from_class[k.intern]
end.pack('U*')
end
end

101
activesupport/test/multibyte_conformance.rb
View File

@@ -1,13 +1,7 @@
require 'abstract_unit'
require 'fileutils'
require 'open-uri'
if RUBY_VERSION < '1.9'
$KCODE = 'UTF8'
UNIDATA_URL = "http://www.unicode.org/Public/#{ActiveSupport::Multibyte::UNICODE_VERSION}/ucd"
UNIDATA_FILE = '/NormalizationTest.txt'
CACHE_DIR = File.dirname(__FILE__) + '/cache'
require 'tmpdir'
class Downloader
def self.download(from, to)
@@ -27,17 +21,19 @@ class Downloader
end
end
class String
# Unicode Inspect returns the codepoints of the string in hex
def ui
"#{self} " + ("[%s]" % unpack("U*").map{|cp| cp.to_s(16) }.join(' '))
end unless ''.respond_to?(:ui)
end
Dir.mkdir(CACHE_DIR) unless File.exist?(CACHE_DIR)
Downloader.download(UNIDATA_URL + UNIDATA_FILE, CACHE_DIR + UNIDATA_FILE)
module ConformanceTest
class MultibyteConformanceTest < Test::Unit::TestCase
include MultibyteTest
UNIDATA_URL = "http://www.unicode.org/Public/#{ActiveSupport::Multibyte::UNICODE_VERSION}/ucd"
UNIDATA_FILE = '/NormalizationTest.txt'
CACHE_DIR = File.join(Dir.tmpdir, 'cache')
def setup
FileUtils.mkdir_p(CACHE_DIR)
Downloader.download(UNIDATA_URL + UNIDATA_FILE, CACHE_DIR + UNIDATA_FILE)
@proxy = ActiveSupport::Multibyte::Chars
end
def test_normalizations_C
each_line_of_norm_tests do |*cols|
col1, col2, col3, col4, col5, comment = *cols
@@ -47,13 +43,13 @@ module ConformanceTest
#
# NFC
# c2 == NFC(c1) == NFC(c2) == NFC(c3)
assert_equal col2.ui, @handler.normalize(col1, :c).ui, "Form C - Col 2 has to be NFC(1) - #{comment}"
assert_equal col2.ui, @handler.normalize(col2, :c).ui, "Form C - Col 2 has to be NFC(2) - #{comment}"
assert_equal col2.ui, @handler.normalize(col3, :c).ui, "Form C - Col 2 has to be NFC(3) - #{comment}"
assert_equal_codepoints col2, @proxy.new(col1).normalize(:c), "Form C - Col 2 has to be NFC(1) - #{comment}"
assert_equal_codepoints col2, @proxy.new(col2).normalize(:c), "Form C - Col 2 has to be NFC(2) - #{comment}"
assert_equal_codepoints col2, @proxy.new(col3).normalize(:c), "Form C - Col 2 has to be NFC(3) - #{comment}"
#
# c4 == NFC(c4) == NFC(c5)
assert_equal col4.ui, @handler.normalize(col4, :c).ui, "Form C - Col 4 has to be C(4) - #{comment}"
assert_equal col4.ui, @handler.normalize(col5, :c).ui, "Form C - Col 4 has to be C(5) - #{comment}"
assert_equal_codepoints col4, @proxy.new(col4).normalize(:c), "Form C - Col 4 has to be C(4) - #{comment}"
assert_equal_codepoints col4, @proxy.new(col5).normalize(:c), "Form C - Col 4 has to be C(5) - #{comment}"
end
end
@@ -63,12 +59,12 @@ module ConformanceTest
#
# NFD
# c3 == NFD(c1) == NFD(c2) == NFD(c3)
assert_equal col3.ui, @handler.normalize(col1, :d).ui, "Form D - Col 3 has to be NFD(1) - #{comment}"
assert_equal col3.ui, @handler.normalize(col2, :d).ui, "Form D - Col 3 has to be NFD(2) - #{comment}"
assert_equal col3.ui, @handler.normalize(col3, :d).ui, "Form D - Col 3 has to be NFD(3) - #{comment}"
assert_equal_codepoints col3, @proxy.new(col1).normalize(:d), "Form D - Col 3 has to be NFD(1) - #{comment}"
assert_equal_codepoints col3, @proxy.new(col2).normalize(:d), "Form D - Col 3 has to be NFD(2) - #{comment}"
assert_equal_codepoints col3, @proxy.new(col3).normalize(:d), "Form D - Col 3 has to be NFD(3) - #{comment}"
# c5 == NFD(c4) == NFD(c5)
assert_equal col5.ui, @handler.normalize(col4, :d).ui, "Form D - Col 5 has to be NFD(4) - #{comment}"
assert_equal col5.ui, @handler.normalize(col5, :d).ui, "Form D - Col 5 has to be NFD(5) - #{comment}"
assert_equal_codepoints col5, @proxy.new(col4).normalize(:d), "Form D - Col 5 has to be NFD(4) - #{comment}"
assert_equal_codepoints col5, @proxy.new(col5).normalize(:d), "Form D - Col 5 has to be NFD(5) - #{comment}"
end
end
@@ -78,11 +74,11 @@ module ConformanceTest
#
# NFKC
# c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
assert_equal col4.ui, @handler.normalize(col1, :kc).ui, "Form D - Col 4 has to be NFKC(1) - #{comment}"
assert_equal col4.ui, @handler.normalize(col2, :kc).ui, "Form D - Col 4 has to be NFKC(2) - #{comment}"
assert_equal col4.ui, @handler.normalize(col3, :kc).ui, "Form D - Col 4 has to be NFKC(3) - #{comment}"
assert_equal col4.ui, @handler.normalize(col4, :kc).ui, "Form D - Col 4 has to be NFKC(4) - #{comment}"
assert_equal col4.ui, @handler.normalize(col5, :kc).ui, "Form D - Col 4 has to be NFKC(5) - #{comment}"
assert_equal_codepoints col4, @proxy.new(col1).normalize(:kc), "Form D - Col 4 has to be NFKC(1) - #{comment}"
assert_equal_codepoints col4, @proxy.new(col2).normalize(:kc), "Form D - Col 4 has to be NFKC(2) - #{comment}"
assert_equal_codepoints col4, @proxy.new(col3).normalize(:kc), "Form D - Col 4 has to be NFKC(3) - #{comment}"
assert_equal_codepoints col4, @proxy.new(col4).normalize(:kc), "Form D - Col 4 has to be NFKC(4) - #{comment}"
assert_equal_codepoints col4, @proxy.new(col5).normalize(:kc), "Form D - Col 4 has to be NFKC(5) - #{comment}"
end
end
@@ -92,11 +88,11 @@ module ConformanceTest
#
# NFKD
# c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
assert_equal col5.ui, @handler.normalize(col1, :kd).ui, "Form KD - Col 5 has to be NFKD(1) - #{comment}"
assert_equal col5.ui, @handler.normalize(col2, :kd).ui, "Form KD - Col 5 has to be NFKD(2) - #{comment}"
assert_equal col5.ui, @handler.normalize(col3, :kd).ui, "Form KD - Col 5 has to be NFKD(3) - #{comment}"
assert_equal col5.ui, @handler.normalize(col4, :kd).ui, "Form KD - Col 5 has to be NFKD(4) - #{comment}"
assert_equal col5.ui, @handler.normalize(col5, :kd).ui, "Form KD - Col 5 has to be NFKD(5) - #{comment}"
assert_equal_codepoints col5, @proxy.new(col1).normalize(:kd), "Form KD - Col 5 has to be NFKD(1) - #{comment}"
assert_equal_codepoints col5, @proxy.new(col2).normalize(:kd), "Form KD - Col 5 has to be NFKD(2) - #{comment}"
assert_equal_codepoints col5, @proxy.new(col3).normalize(:kd), "Form KD - Col 5 has to be NFKD(3) - #{comment}"
assert_equal_codepoints col5, @proxy.new(col4).normalize(:kd), "Form KD - Col 5 has to be NFKD(4) - #{comment}"
assert_equal_codepoints col5, @proxy.new(col5).normalize(:kd), "Form KD - Col 5 has to be NFKD(5) - #{comment}"
end
end
@@ -104,7 +100,7 @@ module ConformanceTest
def each_line_of_norm_tests(&block)
lines = 0
max_test_lines = 0 # Don't limit below 38, because that's the header of the testfile
File.open(File.dirname(__FILE__) + '/cache' + UNIDATA_FILE, 'r') do | f |
File.open(File.join(CACHE_DIR, UNIDATA_FILE), 'r') do | f |
until f.eof? || (max_test_lines > 38 and lines > max_test_lines)
lines += 1
line = f.gets.chomp!
@@ -122,25 +118,8 @@ module ConformanceTest
end
end
end
end
begin
require_library_or_gem('utf8proc_native')
require 'active_record/multibyte/handlers/utf8_handler_proc'
class ConformanceTestProc < Test::Unit::TestCase
include ConformanceTest
def setup
@handler = ::ActiveSupport::Multibyte::Handlers::UTF8HandlerProc
def inspect_codepoints(str)
str.to_s.unpack("U*").map{|cp| cp.to_s(16) }.join(' ')
end
end
rescue LoadError
end
class ConformanceTestPure < Test::Unit::TestCase
include ConformanceTest
def setup
@handler = ::ActiveSupport::Multibyte::Handlers::UTF8Handler
end
end
end
end

372
activesupport/test/multibyte_handler_test.rb
View File

@@ -1,372 +0,0 @@
# encoding: utf-8
require 'abstract_unit'
if RUBY_VERSION < '1.9'
$KCODE = 'UTF8'
class String
# Unicode Inspect returns the codepoints of the string in hex
def ui
"#{self} " + ("[%s]" % unpack("U*").map{|cp| cp.to_s(16) }.join(' '))
end unless ''.respond_to?(:ui)
end
module UTF8HandlingTest
def common_setup
# This is an ASCII string with some russian strings and a ligature. It's nicely calibrated, because
# slicing it at some specific bytes will kill your characters if you use standard Ruby routines.
# It has both capital and standard letters, so that we can test case conversions easily.
# It has 26 characters and 28 when the ligature gets split during normalization.
@string = "Abcd Блå ffi бла бла бла бла"
@string_kd = "Abcd Блå ffi бла бла бла бла"
@string_kc = "Abcd Блå ffi бла бла бла бла"
@string_c = "Abcd Блå ffi бла бла бла бла"
@string_d = "Abcd Блå ffi бла бла бла бла"
@bytestring = "\270\236\010\210\245" # Not UTF-8
# Characters from the character classes as described in UAX #29
@character_from_class = {
:l => 0x1100, :v => 0x1160, :t => 0x11A8, :lv => 0xAC00, :lvt => 0xAC01, :cr => 0x000D, :lf => 0x000A,
:extend => 0x094D, :n => 0x64
}
end
def test_utf8_recognition
assert ActiveSupport::Multibyte::Handlers::UTF8Handler.consumes?(@string),
"Should recognize as a valid UTF-8 string"
assert !ActiveSupport::Multibyte::Handlers::UTF8Handler.consumes?(@bytestring), "This is bytestring, not UTF-8"
end
def test_simple_normalization
# Normalization of DEVANAGARI LETTER QA breaks when composition exclusion isn't used correctly
assert_equal [0x915, 0x93c].pack('U*').ui, [0x915, 0x93c].pack('U*').chars.normalize(:c).to_s.ui
null_byte_str = "Test\0test"
assert_equal '', @handler.normalize(''), "Empty string should not break things"
assert_equal null_byte_str.ui, @handler.normalize(null_byte_str, :kc).ui, "Null byte should remain"
assert_equal null_byte_str.ui, @handler.normalize(null_byte_str, :c).ui, "Null byte should remain"
assert_equal null_byte_str.ui, @handler.normalize(null_byte_str, :d).ui, "Null byte should remain"
assert_equal null_byte_str.ui, @handler.normalize(null_byte_str, :kd).ui, "Null byte should remain"
assert_equal null_byte_str.ui, @handler.decompose(null_byte_str).ui, "Null byte should remain"
assert_equal null_byte_str.ui, @handler.compose(null_byte_str).ui, "Null byte should remain"
comp_str = [
44, # LATIN CAPITAL LETTER D
307, # COMBINING DOT ABOVE
328, # COMBINING OGONEK
323 # COMBINING DOT BELOW
].pack("U*")
norm_str_KC = [44,105,106,328,323].pack("U*")
norm_str_C = [44,307,328,323].pack("U*")
norm_str_D = [44,307,110,780,78,769].pack("U*")
norm_str_KD = [44,105,106,110,780,78,769].pack("U*")
assert_equal norm_str_KC.ui, @handler.normalize(comp_str, :kc).ui, "Should normalize KC"
assert_equal norm_str_C.ui, @handler.normalize(comp_str, :c).ui, "Should normalize C"
assert_equal norm_str_D.ui, @handler.normalize(comp_str, :d).ui, "Should normalize D"
assert_equal norm_str_KD.ui, @handler.normalize(comp_str, :kd).ui, "Should normalize KD"
assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.normalize(@bytestring) }
end
# Test for the Public Review Issue #29, bad explanation of composition might lead to a
# bad implementation: http://www.unicode.org/review/pr-29.html
def test_normalization_C_pri_29
[
[0x0B47, 0x0300, 0x0B3E],
[0x1100, 0x0300, 0x1161]
].map { |c| c.pack('U*') }.each do |c|
assert_equal c.ui, @handler.normalize(c, :c).ui, "Composition is implemented incorrectly"
end
end
def test_casefolding
simple_str = "abCdef"
simple_str_upcase = "ABCDEF"
simple_str_downcase = "abcdef"
assert_equal '', @handler.downcase(@handler.upcase('')), "Empty string should not break things"
assert_equal simple_str_upcase, @handler.upcase(simple_str), "should upcase properly"
assert_equal simple_str_downcase, @handler.downcase(simple_str), "should downcase properly"
assert_equal simple_str_downcase, @handler.downcase(@handler.upcase(simple_str_downcase)), "upcase and downcase should be mirrors"
rus_str = "аБвгд\0f"
rus_str_upcase = "АБВГД\0F"
rus_str_downcase = "абвгд\0f"
assert_equal rus_str_upcase, @handler.upcase(rus_str), "should upcase properly honoring null-byte"
assert_equal rus_str_downcase, @handler.downcase(rus_str), "should downcase properly honoring null-byte"
jap_str = "の埋め込み化対応はほぼ完成"
assert_equal jap_str, @handler.upcase(jap_str), "Japanse has no upcase, should remain unchanged"
assert_equal jap_str, @handler.downcase(jap_str), "Japanse has no downcase, should remain unchanged"
assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.upcase(@bytestring) }
end
def test_capitalize
{ 'аБвг аБвг' => 'Абвг абвг',
'аБвг АБВГ' => 'Абвг абвг',
'АБВГ АБВГ' => 'Абвг абвг',
'' => '' }.each do |f,t|
assert_equal t, @handler.capitalize(f), "Capitalize should work as expected"
end
assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.capitalize(@bytestring) }
end
def test_translate_offset
str = "Блaå" # [2, 2, 1, 2] bytes
assert_equal 0, @handler.translate_offset('', 0), "Offset for an empty string makes no sense, return 0"
assert_equal 0, @handler.translate_offset(str, 0), "First character, first byte"
assert_equal 0, @handler.translate_offset(str, 1), "First character, second byte"
assert_equal 1, @handler.translate_offset(str, 2), "Second character, third byte"
assert_equal 1, @handler.translate_offset(str, 3), "Second character, fourth byte"
assert_equal 2, @handler.translate_offset(str, 4), "Third character, fifth byte"
assert_equal 3, @handler.translate_offset(str, 5), "Fourth character, sixth byte"
assert_equal 3, @handler.translate_offset(str, 6), "Fourth character, seventh byte"
assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.translate_offset(@bytestring, 3) }
end
def test_insert
assert_equal '', @handler.insert('', 0, ''), "Empty string should not break things"
assert_equal "Abcd Блå ffiБУМ бла бла бла бла", @handler.insert(@string, 10, "БУМ"),
"Text should be inserted at right codepoints"
assert_equal "Abcd Блå ffiБУМ бла бла бла бла", @string, "Insert should be destructive"
assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) do
@handler.insert(@bytestring, 2, "\210")
end
end
def test_reverse
str = "wБлåa \n"
rev = "\n aåлБw"
assert_equal '', @handler.reverse(''), "Empty string shouldn't change"
assert_equal rev.ui, @handler.reverse(str).ui, "Should reverse properly"
assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.reverse(@bytestring) }
end
def test_size
assert_equal 0, @handler.size(''), "Empty string has size 0"
assert_equal 26, @handler.size(@string), "String length should be 26"
assert_equal 26, @handler.length(@string), "String length method should be properly aliased"
assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.size(@bytestring) }
end
def test_slice
assert_equal 0x41, @handler.slice(@string, 0), "Singular characters should return codepoints"
assert_equal 0xE5, @handler.slice(@string, 7), "Singular characters should return codepoints"
assert_equal nil, @handler.slice('', -1..1), "Broken range should return nil"
assert_equal '', @handler.slice('', 0..10), "Empty string should not break things"
assert_equal "d Блå ffi", @handler.slice(@string, 3..9), "Unicode characters have to be returned"
assert_equal "d Блå ffi", @handler.slice(@string, 3, 7), "Unicode characters have to be returned"
assert_equal "A", @handler.slice(@string, 0, 1), "Slicing from an offset should return characters"
assert_equal " Блå ffi ", @handler.slice(@string, 4..10), "Unicode characters have to be returned"
assert_equal "ffi бла", @handler.slice(@string, /ffi бла/u), "Slicing on Regexps should be supported"
assert_equal "ffi бла", @handler.slice(@string, /ffi \w\wа/u), "Slicing on Regexps should be supported"
assert_equal nil, @handler.slice(@string, /unknown/u), "Slicing on Regexps with no match should return nil"
assert_equal "ffi бла", @handler.slice(@string, /(ffi бла)/u,1), "Slicing on Regexps with a match group should be supported"
assert_equal nil, @handler.slice(@string, /(ffi)/u,2), "Slicing with a Regexp and asking for an invalid match group should return nil"
assert_equal "", @handler.slice(@string, 7..6), "Range is empty, should return an empty string"
assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.slice(@bytestring, 2..3) }
assert_raise(TypeError, "With 2 args, should raise TypeError for non-Numeric or Regexp first argument") { @handler.slice(@string, 2..3, 1) }
assert_raise(TypeError, "With 2 args, should raise TypeError for non-Numeric or Regexp second argument") { @handler.slice(@string, 1, 2..3) }
assert_raise(ArgumentError, "Should raise ArgumentError when there are more than 2 args") { @handler.slice(@string, 1, 1, 1) }
end
def test_grapheme_cluster_length
assert_equal 0, @handler.g_length(''), "String should count 0 grapheme clusters"
assert_equal 2, @handler.g_length([0x0924, 0x094D, 0x0930].pack('U*')), "String should count 2 grapheme clusters"
assert_equal 1, @handler.g_length(string_from_classes(%w(cr lf))), "Don't cut between CR and LF"
assert_equal 1, @handler.g_length(string_from_classes(%w(l l))), "Don't cut between L"
assert_equal 1, @handler.g_length(string_from_classes(%w(l v))), "Don't cut between L and V"
assert_equal 1, @handler.g_length(string_from_classes(%w(l lv))), "Don't cut between L and LV"
assert_equal 1, @handler.g_length(string_from_classes(%w(l lvt))), "Don't cut between L and LVT"
assert_equal 1, @handler.g_length(string_from_classes(%w(lv v))), "Don't cut between LV and V"
assert_equal 1, @handler.g_length(string_from_classes(%w(lv t))), "Don't cut between LV and T"
assert_equal 1, @handler.g_length(string_from_classes(%w(v v))), "Don't cut between V and V"
assert_equal 1, @handler.g_length(string_from_classes(%w(v t))), "Don't cut between V and T"
assert_equal 1, @handler.g_length(string_from_classes(%w(lvt t))), "Don't cut between LVT and T"
assert_equal 1, @handler.g_length(string_from_classes(%w(t t))), "Don't cut between T and T"
assert_equal 1, @handler.g_length(string_from_classes(%w(n extend))), "Don't cut before Extend"
assert_equal 2, @handler.g_length(string_from_classes(%w(n n))), "Cut between normal characters"
assert_equal 3, @handler.g_length(string_from_classes(%w(n cr lf n))), "Don't cut between CR and LF"
assert_equal 2, @handler.g_length(string_from_classes(%w(n l v t))), "Don't cut between L, V and T"
assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.g_length(@bytestring) }
end
def test_index
s = "Καλημέρα κόσμε!"
assert_equal 0, @handler.index('', ''), "The empty string is always found at the beginning of the string"
assert_equal 0, @handler.index('haystack', ''), "The empty string is always found at the beginning of the string"
assert_equal 0, @handler.index(s, 'Κ'), "Greek K is at 0"
assert_equal 1, @handler.index(s, 'α'), "Greek Alpha is at 1"
assert_equal nil, @handler.index(@bytestring, 'a')
assert_raise(ActiveSupport::Multibyte::Handlers::EncodingError) { @handler.index(@bytestring, "\010") }
end
def test_indexed_insert
s = "Καλη!"
@handler[s, 2] = "a"
assert_equal "Καaη!", s
@handler[s, 2] = "ηη"
assert_equal "Καηηη!", s
assert_raises(IndexError) { @handler[s, 10] = 'a' }
assert_equal "Καηηη!", s
@handler[s, 2] = 32
assert_equal "Κα ηη!", s
@handler[s, 3, 2] = "λλλ"
assert_equal "Κα λλλ!", s
@handler[s, 1, 0] = "λ"
assert_equal "Κλα λλλ!", s
assert_raises(IndexError) { @handler[s, 10, 4] = 'a' }
assert_equal "Κλα λλλ!", s
@handler[s, 4..6] = "ηη"
assert_equal "Κλα ηη!", s
assert_raises(RangeError) { @handler[s, 10..12] = 'a' }
assert_equal "Κλα ηη!", s
@handler[s, /ηη/] = "λλλ"
assert_equal "Κλα λλλ!", s
assert_raises(IndexError) { @handler[s, /ii/] = 'a' }
assert_equal "Κλα λλλ!", s
@handler[s, /(λλ)(.)/, 2] = "α"
assert_equal "Κλα λλα!", s
assert_raises(IndexError) { @handler[s, /()/, 10] = 'a' }
assert_equal "Κλα λλα!", s
@handler[s, "α"] = "η"
assert_equal "Κλη λλα!", s
@handler[s, "λλ"] = "ααα"
assert_equal "Κλη αααα!", s
end
def test_rjust
s = "Καη"
assert_raises(ArgumentError) { @handler.rjust(s, 10, '') }
assert_raises(ArgumentError) { @handler.rjust(s) }
assert_equal "Καη", @handler.rjust(s, -3)
assert_equal "Καη", @handler.rjust(s, 0)
assert_equal "Καη", @handler.rjust(s, 3)
assert_equal " Καη", @handler.rjust(s, 5)
assert_equal " Καη", @handler.rjust(s, 7)
assert_equal "----Καη", @handler.rjust(s, 7, '-')
assert_equal "ααααΚαη", @handler.rjust(s, 7, 'α')
assert_equal "abaΚαη", @handler.rjust(s, 6, 'ab')
assert_equal "αηαΚαη", @handler.rjust(s, 6, 'αη')
end
def test_ljust
s = "Καη"
assert_raises(ArgumentError) { @handler.ljust(s, 10, '') }
assert_raises(ArgumentError) { @handler.ljust(s) }
assert_equal "Καη", @handler.ljust(s, -3)
assert_equal "Καη", @handler.ljust(s, 0)
assert_equal "Καη", @handler.ljust(s, 3)
assert_equal "Καη ", @handler.ljust(s, 5)
assert_equal "Καη ", @handler.ljust(s, 7)
assert_equal "Καη----", @handler.ljust(s, 7, '-')
assert_equal "Καηαααα", @handler.ljust(s, 7, 'α')
assert_equal "Καηaba", @handler.ljust(s, 6, 'ab')
assert_equal "Καηαηα", @handler.ljust(s, 6, 'αη')
end
def test_center
s = "Καη"
assert_raises(ArgumentError) { @handler.center(s, 10, '') }
assert_raises(ArgumentError) { @handler.center(s) }
assert_equal "Καη", @handler.center(s, -3)
assert_equal "Καη", @handler.center(s, 0)
assert_equal "Καη", @handler.center(s, 3)
assert_equal "Καη ", @handler.center(s, 4)
assert_equal " Καη ", @handler.center(s, 5)
assert_equal " Καη ", @handler.center(s, 6)
assert_equal "--Καη--", @handler.center(s, 7, '-')
assert_equal "--Καη---", @handler.center(s, 8, '-')
assert_equal "ααΚαηαα", @handler.center(s, 7, 'α')
assert_equal "ααΚαηααα", @handler.center(s, 8, 'α')
assert_equal "aΚαηab", @handler.center(s, 6, 'ab')
assert_equal "abΚαηab", @handler.center(s, 7, 'ab')
assert_equal "ababΚαηabab", @handler.center(s, 11, 'ab')
assert_equal "αΚαηαη", @handler.center(s, 6, 'αη')
assert_equal "αηΚαηαη", @handler.center(s, 7, 'αη')
end
def test_strip
# A unicode aware version of strip should strip all 26 types of whitespace. This includes the NO BREAK SPACE
# aka BOM (byte order mark). The byte order mark has no place in UTF-8 because it's used to detect LE and BE.
b = "\n" + [
32, # SPACE
8195, # EM SPACE
8199, # FIGURE SPACE,
8201, # THIN SPACE
8202, # HAIR SPACE
65279, # NO BREAK SPACE (ZW)
].pack('U*')
m = "word блин\n\n\n word"
e = [
65279, # NO BREAK SPACE (ZW)
8201, # THIN SPACE
8199, # FIGURE SPACE,
32, # SPACE
].pack('U*')
string = b+m+e
assert_equal '', @handler.strip(''), "Empty string should stay empty"
assert_equal m+e, @handler.lstrip(string), "Whitespace should be gone on the left"
assert_equal b+m, @handler.rstrip(string), "Whitespace should be gone on the right"
assert_equal m, @handler.strip(string), "Whitespace should be stripped on both sides"
bs = "\n #{@bytestring} \n\n"
assert_equal @bytestring, @handler.strip(bs), "Invalid unicode strings should still strip"
end
def test_tidy_bytes
result = [0xb8, 0x17e, 0x8, 0x2c6, 0xa5].pack('U*')
assert_equal result, @handler.tidy_bytes(@bytestring)
assert_equal "a#{result}a", @handler.tidy_bytes('a' + @bytestring + 'a'),
'tidy_bytes should leave surrounding characters intact'
assert_equal "é#{result}é", @handler.tidy_bytes('é' + @bytestring + 'é'),
'tidy_bytes should leave surrounding characters intact'
assert_nothing_raised { @handler.tidy_bytes(@bytestring).unpack('U*') }
assert_equal "\xC3\xA7", @handler.tidy_bytes("\xE7") # iso_8859_1: small c cedilla
assert_equal "\xC2\xA9", @handler.tidy_bytes("\xA9") # iso_8859_1: copyright symbol
assert_equal "\xE2\x80\x9C", @handler.tidy_bytes("\x93") # win_1252: left smart quote
assert_equal "\xE2\x82\xAC", @handler.tidy_bytes("\x80") # win_1252: euro
assert_equal "\x00", @handler.tidy_bytes("\x00") # null char
assert_equal [0xfffd].pack('U'), @handler.tidy_bytes("\xef\xbf\xbd") # invalid char
end
protected
def string_from_classes(classes)
classes.collect do |k|
@character_from_class[k.intern]
end.pack('U*')
end
end
begin
require_library_or_gem('utf8proc_native')
require 'active_record/multibyte/handlers/utf8_handler_proc'
class UTF8HandlingTestProc < Test::Unit::TestCase
include UTF8HandlingTest
def setup
common_setup
@handler = ::ActiveSupport::Multibyte::Handlers::UTF8HandlerProc
end
end
rescue LoadError
end
class UTF8HandlingTestPure < Test::Unit::TestCase
include UTF8HandlingTest
def setup
common_setup
@handler = ::ActiveSupport::Multibyte::Handlers::UTF8Handler
end
end
end

28
activesupport/test/multibyte_unicode_database_test.rb Normal file
View File

@@ -0,0 +1,28 @@
# encoding: utf-8
require 'abstract_unit'
uses_mocha "MultibyteUnicodeDatabaseTest" do
class MultibyteUnicodeDatabaseTest < Test::Unit::TestCase
def setup
@ucd = ActiveSupport::Multibyte::UnicodeDatabase.new
end
ActiveSupport::Multibyte::UnicodeDatabase::ATTRIBUTES.each do |attribute|
define_method "test_lazy_loading_on_attribute_access_of_#{attribute}" do
@ucd.expects(:load)
@ucd.send(attribute)
end
end
def test_load
@ucd.load
ActiveSupport::Multibyte::UnicodeDatabase::ATTRIBUTES.each do |attribute|
assert @ucd.send(attribute).length > 1
end
end
end
end