1 // Copyright 2013 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 //go:generate go run gen.go -output tables.go 6 7 package language 8 9 // TODO: Remove above NOTE after: 10 // - verifying that tables are dropped correctly (most notably matcher tables). 11 12 import ( 13 "strings" 14 15 "golang.org/x/text/internal/language" 16 "golang.org/x/text/internal/language/compact" 17 ) 18 19 // Tag represents a BCP 47 language tag. It is used to specify an instance of a 20 // specific language or locale. All language tag values are guaranteed to be 21 // well-formed. 22 type Tag compact.Tag 23 24 func makeTag(t language.Tag) (tag Tag) { 25 return Tag(compact.Make(t)) 26 } 27 28 func (t *Tag) tag() language.Tag { 29 return (*compact.Tag)(t).Tag() 30 } 31 32 func (t *Tag) isCompact() bool { 33 return (*compact.Tag)(t).IsCompact() 34 } 35 36 // TODO: improve performance. 37 func (t *Tag) lang() language.Language { return t.tag().LangID } 38 func (t *Tag) region() language.Region { return t.tag().RegionID } 39 func (t *Tag) script() language.Script { return t.tag().ScriptID } 40 41 // Make is a convenience wrapper for Parse that omits the error. 42 // In case of an error, a sensible default is returned. 43 func Make(s string) Tag { 44 return Default.Make(s) 45 } 46 47 // Make is a convenience wrapper for c.Parse that omits the error. 48 // In case of an error, a sensible default is returned. 49 func (c CanonType) Make(s string) Tag { 50 t, _ := c.Parse(s) 51 return t 52 } 53 54 // Raw returns the raw base language, script and region, without making an 55 // attempt to infer their values. 56 func (t Tag) Raw() (b Base, s Script, r Region) { 57 tt := t.tag() 58 return Base{tt.LangID}, Script{tt.ScriptID}, Region{tt.RegionID} 59 } 60 61 // IsRoot returns true if t is equal to language "und". 62 func (t Tag) IsRoot() bool { 63 return compact.Tag(t).IsRoot() 64 } 65 66 // CanonType can be used to enable or disable various types of canonicalization. 67 type CanonType int 68 69 const ( 70 // Replace deprecated base languages with their preferred replacements. 71 DeprecatedBase CanonType = 1 << iota 72 // Replace deprecated scripts with their preferred replacements. 73 DeprecatedScript 74 // Replace deprecated regions with their preferred replacements. 75 DeprecatedRegion 76 // Remove redundant scripts. 77 SuppressScript 78 // Normalize legacy encodings. This includes legacy languages defined in 79 // CLDR as well as bibliographic codes defined in ISO-639. 80 Legacy 81 // Map the dominant language of a macro language group to the macro language 82 // subtag. For example cmn -> zh. 83 Macro 84 // The CLDR flag should be used if full compatibility with CLDR is required. 85 // There are a few cases where language.Tag may differ from CLDR. To follow all 86 // of CLDR's suggestions, use All|CLDR. 87 CLDR 88 89 // Raw can be used to Compose or Parse without Canonicalization. 90 Raw CanonType = 0 91 92 // Replace all deprecated tags with their preferred replacements. 93 Deprecated = DeprecatedBase | DeprecatedScript | DeprecatedRegion 94 95 // All canonicalizations recommended by BCP 47. 96 BCP47 = Deprecated | SuppressScript 97 98 // All canonicalizations. 99 All = BCP47 | Legacy | Macro 100 101 // Default is the canonicalization used by Parse, Make and Compose. To 102 // preserve as much information as possible, canonicalizations that remove 103 // potentially valuable information are not included. The Matcher is 104 // designed to recognize similar tags that would be the same if 105 // they were canonicalized using All. 106 Default = Deprecated | Legacy 107 108 canonLang = DeprecatedBase | Legacy | Macro 109 110 // TODO: LikelyScript, LikelyRegion: suppress similar to ICU. 111 ) 112 113 // canonicalize returns the canonicalized equivalent of the tag and 114 // whether there was any change. 115 func canonicalize(c CanonType, t language.Tag) (language.Tag, bool) { 116 if c == Raw { 117 return t, false 118 } 119 changed := false 120 if c&SuppressScript != 0 { 121 if t.LangID.SuppressScript() == t.ScriptID { 122 t.ScriptID = 0 123 changed = true 124 } 125 } 126 if c&canonLang != 0 { 127 for { 128 if l, aliasType := t.LangID.Canonicalize(); l != t.LangID { 129 switch aliasType { 130 case language.Legacy: 131 if c&Legacy != 0 { 132 if t.LangID == _sh && t.ScriptID == 0 { 133 t.ScriptID = _Latn 134 } 135 t.LangID = l 136 changed = true 137 } 138 case language.Macro: 139 if c&Macro != 0 { 140 // We deviate here from CLDR. The mapping "nb" -> "no" 141 // qualifies as a typical Macro language mapping. However, 142 // for legacy reasons, CLDR maps "no", the macro language 143 // code for Norwegian, to the dominant variant "nb". This 144 // change is currently under consideration for CLDR as well. 145 // See https://unicode.org/cldr/trac/ticket/2698 and also 146 // https://unicode.org/cldr/trac/ticket/1790 for some of the 147 // practical implications. TODO: this check could be removed 148 // if CLDR adopts this change. 149 if c&CLDR == 0 || t.LangID != _nb { 150 changed = true 151 t.LangID = l 152 } 153 } 154 case language.Deprecated: 155 if c&DeprecatedBase != 0 { 156 if t.LangID == _mo && t.RegionID == 0 { 157 t.RegionID = _MD 158 } 159 t.LangID = l 160 changed = true 161 // Other canonicalization types may still apply. 162 continue 163 } 164 } 165 } else if c&Legacy != 0 && t.LangID == _no && c&CLDR != 0 { 166 t.LangID = _nb 167 changed = true 168 } 169 break 170 } 171 } 172 if c&DeprecatedScript != 0 { 173 if t.ScriptID == _Qaai { 174 changed = true 175 t.ScriptID = _Zinh 176 } 177 } 178 if c&DeprecatedRegion != 0 { 179 if r := t.RegionID.Canonicalize(); r != t.RegionID { 180 changed = true 181 t.RegionID = r 182 } 183 } 184 return t, changed 185 } 186 187 // Canonicalize returns the canonicalized equivalent of the tag. 188 func (c CanonType) Canonicalize(t Tag) (Tag, error) { 189 // First try fast path. 190 if t.isCompact() { 191 if _, changed := canonicalize(c, compact.Tag(t).Tag()); !changed { 192 return t, nil 193 } 194 } 195 // It is unlikely that one will canonicalize a tag after matching. So do 196 // a slow but simple approach here. 197 if tag, changed := canonicalize(c, t.tag()); changed { 198 tag.RemakeString() 199 return makeTag(tag), nil 200 } 201 return t, nil 202 203 } 204 205 // Confidence indicates the level of certainty for a given return value. 206 // For example, Serbian may be written in Cyrillic or Latin script. 207 // The confidence level indicates whether a value was explicitly specified, 208 // whether it is typically the only possible value, or whether there is 209 // an ambiguity. 210 type Confidence int 211 212 const ( 213 No Confidence = iota // full confidence that there was no match 214 Low // most likely value picked out of a set of alternatives 215 High // value is generally assumed to be the correct match 216 Exact // exact match or explicitly specified value 217 ) 218 219 var confName = []string{"No", "Low", "High", "Exact"} 220 221 func (c Confidence) String() string { 222 return confName[c] 223 } 224 225 // String returns the canonical string representation of the language tag. 226 func (t Tag) String() string { 227 return t.tag().String() 228 } 229 230 // MarshalText implements encoding.TextMarshaler. 231 func (t Tag) MarshalText() (text []byte, err error) { 232 return t.tag().MarshalText() 233 } 234 235 // UnmarshalText implements encoding.TextUnmarshaler. 236 func (t *Tag) UnmarshalText(text []byte) error { 237 var tag language.Tag 238 err := tag.UnmarshalText(text) 239 *t = makeTag(tag) 240 return err 241 } 242 243 // Base returns the base language of the language tag. If the base language is 244 // unspecified, an attempt will be made to infer it from the context. 245 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change. 246 func (t Tag) Base() (Base, Confidence) { 247 if b := t.lang(); b != 0 { 248 return Base{b}, Exact 249 } 250 tt := t.tag() 251 c := High 252 if tt.ScriptID == 0 && !tt.RegionID.IsCountry() { 253 c = Low 254 } 255 if tag, err := tt.Maximize(); err == nil && tag.LangID != 0 { 256 return Base{tag.LangID}, c 257 } 258 return Base{0}, No 259 } 260 261 // Script infers the script for the language tag. If it was not explicitly given, it will infer 262 // a most likely candidate. 263 // If more than one script is commonly used for a language, the most likely one 264 // is returned with a low confidence indication. For example, it returns (Cyrl, Low) 265 // for Serbian. 266 // If a script cannot be inferred (Zzzz, No) is returned. We do not use Zyyy (undetermined) 267 // as one would suspect from the IANA registry for BCP 47. In a Unicode context Zyyy marks 268 // common characters (like 1, 2, 3, '.', etc.) and is therefore more like multiple scripts. 269 // See https://www.unicode.org/reports/tr24/#Values for more details. Zzzz is also used for 270 // unknown value in CLDR. (Zzzz, Exact) is returned if Zzzz was explicitly specified. 271 // Note that an inferred script is never guaranteed to be the correct one. Latin is 272 // almost exclusively used for Afrikaans, but Arabic has been used for some texts 273 // in the past. Also, the script that is commonly used may change over time. 274 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change. 275 func (t Tag) Script() (Script, Confidence) { 276 if scr := t.script(); scr != 0 { 277 return Script{scr}, Exact 278 } 279 tt := t.tag() 280 sc, c := language.Script(_Zzzz), No 281 if scr := tt.LangID.SuppressScript(); scr != 0 { 282 // Note: it is not always the case that a language with a suppress 283 // script value is only written in one script (e.g. kk, ms, pa). 284 if tt.RegionID == 0 { 285 return Script{scr}, High 286 } 287 sc, c = scr, High 288 } 289 if tag, err := tt.Maximize(); err == nil { 290 if tag.ScriptID != sc { 291 sc, c = tag.ScriptID, Low 292 } 293 } else { 294 tt, _ = canonicalize(Deprecated|Macro, tt) 295 if tag, err := tt.Maximize(); err == nil && tag.ScriptID != sc { 296 sc, c = tag.ScriptID, Low 297 } 298 } 299 return Script{sc}, c 300 } 301 302 // Region returns the region for the language tag. If it was not explicitly given, it will 303 // infer a most likely candidate from the context. 304 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change. 305 func (t Tag) Region() (Region, Confidence) { 306 if r := t.region(); r != 0 { 307 return Region{r}, Exact 308 } 309 tt := t.tag() 310 if tt, err := tt.Maximize(); err == nil { 311 return Region{tt.RegionID}, Low // TODO: differentiate between high and low. 312 } 313 tt, _ = canonicalize(Deprecated|Macro, tt) 314 if tag, err := tt.Maximize(); err == nil { 315 return Region{tag.RegionID}, Low 316 } 317 return Region{_ZZ}, No // TODO: return world instead of undetermined? 318 } 319 320 // Variants returns the variants specified explicitly for this language tag. 321 // or nil if no variant was specified. 322 func (t Tag) Variants() []Variant { 323 if !compact.Tag(t).MayHaveVariants() { 324 return nil 325 } 326 v := []Variant{} 327 x, str := "", t.tag().Variants() 328 for str != "" { 329 x, str = nextToken(str) 330 v = append(v, Variant{x}) 331 } 332 return v 333 } 334 335 // Parent returns the CLDR parent of t. In CLDR, missing fields in data for a 336 // specific language are substituted with fields from the parent language. 337 // The parent for a language may change for newer versions of CLDR. 338 // 339 // Parent returns a tag for a less specific language that is mutually 340 // intelligible or Und if there is no such language. This may not be the same as 341 // simply stripping the last BCP 47 subtag. For instance, the parent of "zh-TW" 342 // is "zh-Hant", and the parent of "zh-Hant" is "und". 343 func (t Tag) Parent() Tag { 344 return Tag(compact.Tag(t).Parent()) 345 } 346 347 // nextToken returns token t and the rest of the string. 348 func nextToken(s string) (t, tail string) { 349 p := strings.Index(s[1:], "-") 350 if p == -1 { 351 return s[1:], "" 352 } 353 p++ 354 return s[1:p], s[p:] 355 } 356 357 // Extension is a single BCP 47 extension. 358 type Extension struct { 359 s string 360 } 361 362 // String returns the string representation of the extension, including the 363 // type tag. 364 func (e Extension) String() string { 365 return e.s 366 } 367 368 // ParseExtension parses s as an extension and returns it on success. 369 func ParseExtension(s string) (e Extension, err error) { 370 ext, err := language.ParseExtension(s) 371 return Extension{ext}, err 372 } 373 374 // Type returns the one-byte extension type of e. It returns 0 for the zero 375 // exception. 376 func (e Extension) Type() byte { 377 if e.s == "" { 378 return 0 379 } 380 return e.s[0] 381 } 382 383 // Tokens returns the list of tokens of e. 384 func (e Extension) Tokens() []string { 385 return strings.Split(e.s, "-") 386 } 387 388 // Extension returns the extension of type x for tag t. It will return 389 // false for ok if t does not have the requested extension. The returned 390 // extension will be invalid in this case. 391 func (t Tag) Extension(x byte) (ext Extension, ok bool) { 392 if !compact.Tag(t).MayHaveExtensions() { 393 return Extension{}, false 394 } 395 e, ok := t.tag().Extension(x) 396 return Extension{e}, ok 397 } 398 399 // Extensions returns all extensions of t. 400 func (t Tag) Extensions() []Extension { 401 if !compact.Tag(t).MayHaveExtensions() { 402 return nil 403 } 404 e := []Extension{} 405 for _, ext := range t.tag().Extensions() { 406 e = append(e, Extension{ext}) 407 } 408 return e 409 } 410 411 // TypeForKey returns the type associated with the given key, where key and type 412 // are of the allowed values defined for the Unicode locale extension ('u') in 413 // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers. 414 // TypeForKey will traverse the inheritance chain to get the correct value. 415 // 416 // If there are multiple types associated with a key, only the first will be 417 // returned. If there is no type associated with a key, it returns the empty 418 // string. 419 func (t Tag) TypeForKey(key string) string { 420 if !compact.Tag(t).MayHaveExtensions() { 421 if key != "rg" && key != "va" { 422 return "" 423 } 424 } 425 return t.tag().TypeForKey(key) 426 } 427 428 // SetTypeForKey returns a new Tag with the key set to type, where key and type 429 // are of the allowed values defined for the Unicode locale extension ('u') in 430 // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers. 431 // An empty value removes an existing pair with the same key. 432 func (t Tag) SetTypeForKey(key, value string) (Tag, error) { 433 tt, err := t.tag().SetTypeForKey(key, value) 434 return makeTag(tt), err 435 } 436 437 // NumCompactTags is the number of compact tags. The maximum tag is 438 // NumCompactTags-1. 439 const NumCompactTags = compact.NumCompactTags 440 441 // CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags 442 // for which data exists in the text repository.The index will change over time 443 // and should not be stored in persistent storage. If t does not match a compact 444 // index, exact will be false and the compact index will be returned for the 445 // first match after repeatedly taking the Parent of t. 446 func CompactIndex(t Tag) (index int, exact bool) { 447 id, exact := compact.LanguageID(compact.Tag(t)) 448 return int(id), exact 449 } 450 451 var root = language.Tag{} 452 453 // Base is an ISO 639 language code, used for encoding the base language 454 // of a language tag. 455 type Base struct { 456 langID language.Language 457 } 458 459 // ParseBase parses a 2- or 3-letter ISO 639 code. 460 // It returns a ValueError if s is a well-formed but unknown language identifier 461 // or another error if another error occurred. 462 func ParseBase(s string) (Base, error) { 463 l, err := language.ParseBase(s) 464 return Base{l}, err 465 } 466 467 // String returns the BCP 47 representation of the base language. 468 func (b Base) String() string { 469 return b.langID.String() 470 } 471 472 // ISO3 returns the ISO 639-3 language code. 473 func (b Base) ISO3() string { 474 return b.langID.ISO3() 475 } 476 477 // IsPrivateUse reports whether this language code is reserved for private use. 478 func (b Base) IsPrivateUse() bool { 479 return b.langID.IsPrivateUse() 480 } 481 482 // Script is a 4-letter ISO 15924 code for representing scripts. 483 // It is idiomatically represented in title case. 484 type Script struct { 485 scriptID language.Script 486 } 487 488 // ParseScript parses a 4-letter ISO 15924 code. 489 // It returns a ValueError if s is a well-formed but unknown script identifier 490 // or another error if another error occurred. 491 func ParseScript(s string) (Script, error) { 492 sc, err := language.ParseScript(s) 493 return Script{sc}, err 494 } 495 496 // String returns the script code in title case. 497 // It returns "Zzzz" for an unspecified script. 498 func (s Script) String() string { 499 return s.scriptID.String() 500 } 501 502 // IsPrivateUse reports whether this script code is reserved for private use. 503 func (s Script) IsPrivateUse() bool { 504 return s.scriptID.IsPrivateUse() 505 } 506 507 // Region is an ISO 3166-1 or UN M.49 code for representing countries and regions. 508 type Region struct { 509 regionID language.Region 510 } 511 512 // EncodeM49 returns the Region for the given UN M.49 code. 513 // It returns an error if r is not a valid code. 514 func EncodeM49(r int) (Region, error) { 515 rid, err := language.EncodeM49(r) 516 return Region{rid}, err 517 } 518 519 // ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code. 520 // It returns a ValueError if s is a well-formed but unknown region identifier 521 // or another error if another error occurred. 522 func ParseRegion(s string) (Region, error) { 523 r, err := language.ParseRegion(s) 524 return Region{r}, err 525 } 526 527 // String returns the BCP 47 representation for the region. 528 // It returns "ZZ" for an unspecified region. 529 func (r Region) String() string { 530 return r.regionID.String() 531 } 532 533 // ISO3 returns the 3-letter ISO code of r. 534 // Note that not all regions have a 3-letter ISO code. 535 // In such cases this method returns "ZZZ". 536 func (r Region) ISO3() string { 537 return r.regionID.ISO3() 538 } 539 540 // M49 returns the UN M.49 encoding of r, or 0 if this encoding 541 // is not defined for r. 542 func (r Region) M49() int { 543 return r.regionID.M49() 544 } 545 546 // IsPrivateUse reports whether r has the ISO 3166 User-assigned status. This 547 // may include private-use tags that are assigned by CLDR and used in this 548 // implementation. So IsPrivateUse and IsCountry can be simultaneously true. 549 func (r Region) IsPrivateUse() bool { 550 return r.regionID.IsPrivateUse() 551 } 552 553 // IsCountry returns whether this region is a country or autonomous area. This 554 // includes non-standard definitions from CLDR. 555 func (r Region) IsCountry() bool { 556 return r.regionID.IsCountry() 557 } 558 559 // IsGroup returns whether this region defines a collection of regions. This 560 // includes non-standard definitions from CLDR. 561 func (r Region) IsGroup() bool { 562 return r.regionID.IsGroup() 563 } 564 565 // Contains returns whether Region c is contained by Region r. It returns true 566 // if c == r. 567 func (r Region) Contains(c Region) bool { 568 return r.regionID.Contains(c.regionID) 569 } 570 571 // TLD returns the country code top-level domain (ccTLD). UK is returned for GB. 572 // In all other cases it returns either the region itself or an error. 573 // 574 // This method may return an error for a region for which there exists a 575 // canonical form with a ccTLD. To get that ccTLD canonicalize r first. The 576 // region will already be canonicalized it was obtained from a Tag that was 577 // obtained using any of the default methods. 578 func (r Region) TLD() (Region, error) { 579 tld, err := r.regionID.TLD() 580 return Region{tld}, err 581 } 582 583 // Canonicalize returns the region or a possible replacement if the region is 584 // deprecated. It will not return a replacement for deprecated regions that 585 // are split into multiple regions. 586 func (r Region) Canonicalize() Region { 587 return Region{r.regionID.Canonicalize()} 588 } 589 590 // Variant represents a registered variant of a language as defined by BCP 47. 591 type Variant struct { 592 variant string 593 } 594 595 // ParseVariant parses and returns a Variant. An error is returned if s is not 596 // a valid variant. 597 func ParseVariant(s string) (Variant, error) { 598 v, err := language.ParseVariant(s) 599 return Variant{v.String()}, err 600 } 601 602 // String returns the string representation of the variant. 603 func (v Variant) String() string { 604 return v.variant 605 } 606