#!/usr/bin/env perl use utf8; # so literals and identifiers can be in UTF-8 use v5.12; # or later to get "unicode_strings" feature use strict; # quote strings, declare variables use warnings; # on by default use warnings qw(FATAL utf8); # fatalize encoding glitches use open qw(:std :encoding(UTF-8)); # undeclared streams in UTF-8 use charnames qw(:full :short); # unneeded in v5.16
This does make even Unix programmers "binmode" your binary streams, or open them with ":raw", but that's the only way to get at them portably anyway.
WARNING: "use autodie" (pre 2.26) and "use open" do not get along with each other.
use Unicode::Normalize; while (<>) { $_ = NFD($_); # decompose + reorder canonically ... } continue { print NFC($_); # recompose (where possible) + reorder canonically }
use v5.14; # subwarnings unavailable any earlier no warnings "nonchar"; # the 66 forbidden non-characters no warnings "surrogate"; # UTF-16/CESU-8 nonsense no warnings "non_unicode"; # for codepoints over 0x10_FFFF
use utf8; my $measure = "A°ngstro.m"; my @Xsoft = qw( cp852 cp1251 cp1252 ); my @XXXXXXXXX = qw( XXXX XXXXX ); my @X = qw( koi8-f koi8-u koi8-r ); my $motto = "X X X"; # FAMILY, GROWING HEART, DROMEDARY CAMEL
If you forget "use utf8", high bytes will be misunderstood as separate characters, and nothing will work right.
# ASCII characters ord("A") chr(65) # characters from the Basic Multilingual Plane ord("X") chr(0x3A3) # beyond the BMP ord("X") # MATHEMATICAL ITALIC SMALL N chr(0x1D45B) # beyond Unicode! (up to MAXINT) ord("\x{20_0000}") chr(0x20_0000)
String: "\x{3a3}" Regex: /\x{3a3}/ String: "\x{1d45b}" Regex: /\x{1d45b}/ # even non-BMP ranges in regex work fine /[\x{1D434}-\x{1D467}]/
use charnames (); my $name = charnames::viacode(0x03A3);
use charnames (); my $number = charnames::vianame("GREEK CAPITAL LETTER SIGMA");
use charnames qw(:full :short);
But prior to v5.16, you must be explicit about which set of charnames you want. The ":full" names are the official Unicode character name, alias, or sequence, which all share a namespace.
use charnames qw(:full :short latin greek); "\N{MATHEMATICAL ITALIC SMALL N}" # :full "\N{GREEK CAPITAL LETTER SIGMA}" # :full
Anything else is a Perl-specific convenience abbreviation. Specify one or more scripts by names if you want short names that are script-specific.
"\N{Greek:Sigma}" # :short "\N{ae}" # latin "\N{epsilon}" # greek
The v5.16 release also supports a ":loose" import for loose matching of character names, which works just like loose matching of property names: that is, it disregards case, whitespace, and underscores:
"\N{euro sign}" # :loose (from v5.16)
Starting in v5.32, you can also use
qr/\p{name=euro sign}/
to get official Unicode named characters in regular expressions. Loose matching is always done for these.
use charnames qw(:full); my $seq = "\N{LATIN CAPITAL LETTER A WITH MACRON AND GRAVE}"; printf "U+%v04X\n", $seq; U+0100.0300
use charnames ":full", ":alias" => { ecute => "LATIN SMALL LETTER E WITH ACUTE", "APPLE LOGO" => 0xF8FF, # private use character }; "\N{ecute}" "\N{APPLE LOGO}"
# cpan -i Unicode::Unihan use Unicode::Unihan; my $str = "XX"; my $unhan = Unicode::Unihan->new; for my $lang (qw(Mandarin Cantonese Korean JapaneseOn JapaneseKun)) { printf "CJK $str in %-12s is ", $lang; say $unhan->$lang($str); }
prints:
CJK XX in Mandarin is DONG1JING1 CJK XX in Cantonese is dung1ging1 CJK XX in Korean is TONGKYENG CJK XX in JapaneseOn is TOUKYOU KEI KIN CJK XX in JapaneseKun is HIGASHI AZUMAMIYAKO
If you have a specific romanization scheme in mind, use the specific module:
# cpan -i Lingua::JA::Romanize::Japanese use Lingua::JA::Romanize::Japanese; my $k2r = Lingua::JA::Romanize::Japanese->new; my $str = "XX"; say "Japanese for $str is ", $k2r->chars($str);
prints
Japanese for XX is toukyou
use Encode qw(encode decode); my $chars = decode("shiftjis", $bytes, 1); # OR my $bytes = encode("MIME-Header-ISO_2022_JP", $chars, 1);
For streams all in the same encoding, don't use encode/decode; instead set the file encoding when you open the file or immediately after with "binmode" as described later below.
$ perl -CA ... or $ export PERL_UNICODE=A or use Encode qw(decode); @ARGV = map { decode('UTF-8', $_, 1) } @ARGV;
# cpan -i Encode::Locale use Encode qw(locale); use Encode::Locale; # use "locale" as an arg to encode/decode @ARGV = map { decode(locale => $_, 1) } @ARGV;
$ perl -CS ... or $ export PERL_UNICODE=S or use open qw(:std :encoding(UTF-8)); or binmode(STDIN, ":encoding(UTF-8)"); binmode(STDOUT, ":utf8"); binmode(STDERR, ":utf8");
# cpan -i Encode::Locale use Encode; use Encode::Locale; # or as a stream for binmode or open binmode STDIN, ":encoding(console_in)" if -t STDIN; binmode STDOUT, ":encoding(console_out)" if -t STDOUT; binmode STDERR, ":encoding(console_out)" if -t STDERR;
$ perl -CD ... or $ export PERL_UNICODE=D or use open qw(:encoding(UTF-8));
$ perl -CSDA ... or $ export PERL_UNICODE=SDA or use open qw(:std :encoding(UTF-8)); use Encode qw(decode); @ARGV = map { decode('UTF-8', $_, 1) } @ARGV;
# input file open(my $in_file, "< :encoding(UTF-16)", "wintext"); OR open(my $in_file, "<", "wintext"); binmode($in_file, ":encoding(UTF-16)"); THEN my $line = <$in_file>; # output file open($out_file, "> :encoding(cp1252)", "wintext"); OR open(my $out_file, ">", "wintext"); binmode($out_file, ":encoding(cp1252)"); THEN print $out_file "some text\n";
More layers than just the encoding can be specified here. For example, the incantation ":raw :encoding(UTF-16LE) :crlf" includes implicit CRLF handling.
uc("henry X") # "HENRY X" uc("tschu.β") # "TSCHU.SS" notice β => SS # both are true: "tschu.β" =~ /TSCHU.SS/i # notice β => SS "XXXXXXX" =~ /XXXXXXX/i # notice X,X,X sameness
use feature "fc"; # fc() function is from v5.16 # sort case-insensitively my @sorted = sort { fc($a) cmp fc($b) } @list; # both are true: fc("tschu.β") eq fc("TSCHU.SS") fc("XXXXXXX") eq fc("XXXXXXX")
\R s/\R/\n/g; # normalize all linebreaks to \n
use Unicode::UCD qw(charinfo); my $cat = charinfo(0x3A3)->{category}; # "Lu"
use v5.14; use re "/a"; # OR my($num) = $str =~ /(\d+)/a;
Or use specific un-Unicode properties, like "\p{ahex}" and "\p{POSIX_Digit"}. Properties still work normally no matter what charset modifiers ("/d /u /l /a /aa") should be effect.
\pL, \pN, \pS, \pP, \pM, \pZ, \pC \p{Sk}, \p{Ps}, \p{Lt} \p{alpha}, \p{upper}, \p{lower} \p{Latin}, \p{Greek} \p{script_extensions=Latin}, \p{scx=Greek} \p{East_Asian_Width=Wide}, \p{EA=W} \p{Line_Break=Hyphen}, \p{LB=HY} \p{Numeric_Value=4}, \p{NV=4}
# using private-use characters sub In_Tengwar { "E000\tE07F\n" } if (/\p{In_Tengwar}/) { ... } # blending existing properties sub Is_GraecoRoman_Title {<<'END_OF_SET'} +utf8::IsLatin +utf8::IsGreek &utf8::IsTitle END_OF_SET if (/\p{Is_GraecoRoman_Title}/ { ... }
use Unicode::Normalize; my $nfd = NFD($orig); my $nfc = NFC($orig); my $nfkd = NFKD($orig); my $nfkc = NFKC($orig);
use v5.14; # needed for num() function use Unicode::UCD qw(num); my $str = "got X and XXXX and X and here"; my @nums = (); while ($str =~ /(\d+|\N)/g) { # not just ASCII! push @nums, num($1); } say "@nums"; # 12 4567 0.875 use charnames qw(:full); my $nv = num("\N{RUMI DIGIT ONE}\N{RUMI DIGIT TWO}");
# Find vowel *plus* any combining diacritics,underlining,etc. my $nfd = NFD($orig); $nfd =~ / (?=[aeiou]) \X /xi
# match and grab five first graphemes my($first_five) = $str =~ /^ ( \X{5} ) /x;
# cpan -i Unicode::GCString use Unicode::GCString; my $gcs = Unicode::GCString->new($str); my $first_five = $gcs->substr(0, 5);
$str = join("", reverse $str =~ /\X/g); # OR: cpan -i Unicode::GCString use Unicode::GCString; $str = reverse Unicode::GCString->new($str);
my $str = "brulee"; my $count = 0; while ($str =~ /\X/g) { $count++ } # OR: cpan -i Unicode::GCString use Unicode::GCString; my $gcs = Unicode::GCString->new($str); my $count = $gcs->length;
use Unicode::GCString; use Unicode::Normalize; my @words = qw/creme brulee/; @words = map { NFC($_), NFD($_) } @words; for my $str (@words) { my $gcs = Unicode::GCString->new($str); my $cols = $gcs->columns; my $pad = " " x (10 - $cols); say str, $pad, " |"; }
generates this to show that it pads correctly no matter the normalization:
creme | creXme | brulee | bruXleXe |
use Unicode::Collate; my $col = Unicode::Collate->new(); my @list = $col->sort(@old_list);
See the ucsort program from the Unicode::Tussle CPAN module for a convenient command-line interface to this module.
use Unicode::Collate; my $col = Unicode::Collate->new(level => 1); my @list = $col->sort(@old_list);
# either use v5.12, OR: cpan -i Unicode::Collate::Locale use Unicode::Collate::Locale; my $col = Unicode::Collate::Locale->new(locale => "de__phonebook"); my @list = $col->sort(@old_list);
The ucsort program mentioned above accepts a "--locale" parameter.
@srecs = sort { $b->{AGE} <=> $a->{AGE} || $a->{NAME} cmp $b->{NAME} } @recs;
Use this:
my $coll = Unicode::Collate->new(); for my $rec (@recs) { $rec->{NAME_key} = $coll->getSortKey( $rec->{NAME} ); } @srecs = sort { $b->{AGE} <=> $a->{AGE} || $a->{NAME_key} cmp $b->{NAME_key} } @recs;
use Unicode::Collate; my $es = Unicode::Collate->new( level => 1, normalization => undef ); # now both are true: $es->eq("Garcia", "GARCIA" ); $es->eq("Marquez", "MARQUEZ");
my $de = Unicode::Collate::Locale->new( locale => "de__phonebook", ); # now this is true: $de->eq("tschu.β", "TSCHUESS"); # notice u. => UE, β => SS
# cpan -i Unicode::LineBreak use Unicode::LineBreak; use charnames qw(:full); my $para = "This is a super\N{HYPHEN}long string. " x 20; my $fmt = Unicode::LineBreak->new; print $fmt->break($para), "\n";
use DB_File; use Encode qw(encode decode); tie %dbhash, "DB_File", "pathname"; # STORE # assume $uni_key and $uni_value are abstract Unicode strings my $enc_key = encode("UTF-8", $uni_key, 1); my $enc_value = encode("UTF-8", $uni_value, 1); $dbhash{$enc_key} = $enc_value; # FETCH # assume $uni_key holds a normal Perl string (abstract Unicode) my $enc_key = encode("UTF-8", $uni_key, 1); my $enc_value = $dbhash{$enc_key}; my $uni_value = decode("UTF-8", $enc_value, 1);
use DB_File; use DBM_Filter; my $dbobj = tie %dbhash, "DB_File", "pathname"; $dbobj->Filter_Value("utf8"); # this is the magic bit # STORE # assume $uni_key and $uni_value are abstract Unicode strings $dbhash{$uni_key} = $uni_value; # FETCH # $uni_key holds a normal Perl string (abstract Unicode) my $uni_value = $dbhash{$uni_key};
Creme Brulee....... X2.00 Eclair............. X1.60 Fideua............. X4.20 Hamburger.......... X6.00 Jamon Serrano...... X4.45 Linguica........... X7.00 Pate............... X4.15 Pears.............. X2.00 Peches............. X2.25 Smorbrod........... X5.75 Spa.tzle............ X5.50 Xorico............. X3.00 XXXXX.............. X6.50 XXX............. X4.00 XXX............. X2.65 XXXXX......... X8.00 XXXXXXX..... X1.85 XX............... X9.99 XX............... X7.50
Here's that program; tested on v5.14.
#!/usr/bin/env perl # umenu - demo sorting and printing of Unicode food # # (obligatory and increasingly long preamble) # use utf8; use v5.14; # for locale sorting use strict; use warnings; use warnings qw(FATAL utf8); # fatalize encoding faults use open qw(:std :encoding(UTF-8)); # undeclared streams in UTF-8 use charnames qw(:full :short); # unneeded in v5.16 # std modules use Unicode::Normalize; # std perl distro as of v5.8 use List::Util qw(max); # std perl distro as of v5.10 use Unicode::Collate::Locale; # std perl distro as of v5.14 # cpan modules use Unicode::GCString; # from CPAN # forward defs sub pad($$$); sub colwidth(_); sub entitle(_); my %price = ( "XXXXX" => 6.50, # gyros "pears" => 2.00, # like um, pears "linguica" => 7.00, # spicy sausage, Portuguese "xorico" => 3.00, # chorizo sausage, Catalan "hamburger" => 6.00, # burgermeister meisterburger "eclair" => 1.60, # dessert, French "smorbrod" => 5.75, # sandwiches, Norwegian "spa.tzle" => 5.50, # Bayerisch noodles, little sparrows "XX" => 7.50, # bao1 zi5, steamed pork buns, Mandarin "jamon serrano" => 4.45, # country ham, Spanish "peches" => 2.25, # peaches, French "XXXXXXX" => 1.85, # cream-filled pastry like eclair "XXX" => 4.00, # makgeolli, Korean rice wine "XX" => 9.99, # sushi, Japanese "XXX" => 2.65, # omochi, rice cakes, Japanese "creme brulee" => 2.00, # crema catalana "fideua" => 4.20, # more noodles, Valencian # (Catalan=fideuada) "pate" => 4.15, # gooseliver paste, French "XXXXX" => 8.00, # okonomiyaki, Japanese ); my $width = 5 + max map { colwidth } keys %price; # So the Asian stuff comes out in an order that someone # who reads those scripts won't freak out over; the # CJK stuff will be in JIS X 0208 order that way. my $coll = Unicode::Collate::Locale->new(locale => "ja"); for my $item ($coll->sort(keys %price)) { print pad(entitle($item), $width, "."); printf " X%.2f\n", $price{$item}; } sub pad($$$) { my($str, $width, $padchar) = @_; return $str . ($padchar x ($width - colwidth($str))); } sub colwidth(_) { my($str) = @_; return Unicode::GCString->new($str)->columns; } sub entitle(_) { my($str) = @_; $str =~ s{ (?=\pL)(\S) (\S*) } { ucfirst($1) . lc($2) }xge; return $str; }
The Unicode::Tussle CPAN module includes many programs to help with working with Unicode, including these programs to fully or partly replace standard utilities: tcgrep instead of egrep, uniquote instead of cat -v or hexdump, uniwc instead of wc, unilook instead of look, unifmt instead of fmt, and ucsort instead of sort. For exploring Unicode character names and character properties, see its uniprops, unichars, and uninames programs. It also supplies these programs, all of which are general filters that do Unicode-y things: unititle and unicaps; uniwide and uninarrow; unisupers and unisubs; nfd, nfc, nfkd, and nfkc; and uc, lc, and tc.
Finally, see the published Unicode Standard (page numbers are from version 6.0.0), including these specific annexes and technical reports:
This program is free software; you may redistribute it and/or modify it under the same terms as Perl itself.
Most of these examples taken from the current edition of the XCamel BookX; that is, from the 4XX Edition of Programming Perl, Copyright X 2012 Tom Christiansen <et al.>, 2012-02-13 by OXReilly Media. The code itself is freely redistributable, and you are encouraged to transplant, fold, spindle, and mutilate any of the examples in this manpage however you please for inclusion into your own programs without any encumbrance whatsoever. Acknowledgement via code comment is polite but not required.