module ActiveSupport::Multibyte::Unicode

Constants

UNICODE_VERSION: The Unicode version that is supported by the implementation

Public Instance Methods

compose(codepoints) Show source

# File activesupport/lib/active_support/multibyte/unicode.rb, line 33
def compose(codepoints)
  codepoints.pack("U*").unicode_normalize(:nfc).codepoints
end

Compose decomposed characters to the composed form.

decompose(type, codepoints) Show source

# File activesupport/lib/active_support/multibyte/unicode.rb, line 24
def decompose(type, codepoints)
  if type == :compatibility
    codepoints.pack("U*").unicode_normalize(:nfkd).codepoints
  else
    codepoints.pack("U*").unicode_normalize(:nfd).codepoints
  end
end

Decompose composed characters to the decomposed form.

default_normalization_form() Show source

# File activesupport/lib/active_support/multibyte/unicode.rb, line 11
def default_normalization_form
  ActiveSupport::Deprecation.warn(
    "ActiveSupport::Multibyte::Unicode.default_normalization_form is deprecated and will be removed in Rails 6.2."
  )
end

default_normalization_form=(_) Show source

# File activesupport/lib/active_support/multibyte/unicode.rb, line 17
def default_normalization_form=(_)
  ActiveSupport::Deprecation.warn(
    "ActiveSupport::Multibyte::Unicode.default_normalization_form= is deprecated and will be removed in Rails 6.2."
  )
end

tidy_bytes(string, force = false) Show source

# File activesupport/lib/active_support/multibyte/unicode.rb, line 44
def tidy_bytes(string, force = false)
  return string if string.empty? || string.ascii_only?
  return recode_windows1252_chars(string) if force
  string.scrub { |bad| recode_windows1252_chars(bad) }
end

Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent resulting in a valid UTF-8 string.

Passing true will forcibly tidy all bytes, assuming that the string's encoding is entirely CP1252 or ISO-8859-1.