1
// Copyright 2013 The Go Authors. All rights reserved.
2
// Use of this source code is governed by a BSD-style
3
// license that can be found in the LICENSE file.
4

5
//go:generate go run gen.go -output tables.go
6

7
package language
8

9
// TODO: Remove above NOTE after:
10
// - verifying that tables are dropped correctly (most notably matcher tables).
11

12
import (
13
	"strings"
14

15
	"golang.org/x/text/internal/language"
16
	"golang.org/x/text/internal/language/compact"
17
)
18

19
// Tag represents a BCP 47 language tag. It is used to specify an instance of a
20
// specific language or locale. All language tag values are guaranteed to be
21
// well-formed.
22
type Tag compact.Tag
23

24
func makeTag(t language.Tag) (tag Tag) {
25
	return Tag(compact.Make(t))
26
}
27

28
func (t *Tag) tag() language.Tag {
29
	return (*compact.Tag)(t).Tag()
30
}
31

32
func (t *Tag) isCompact() bool {
33
	return (*compact.Tag)(t).IsCompact()
34
}
35

36
// TODO: improve performance.
37
func (t *Tag) lang() language.Language { return t.tag().LangID }
38
func (t *Tag) region() language.Region { return t.tag().RegionID }
39
func (t *Tag) script() language.Script { return t.tag().ScriptID }
40

41
// Make is a convenience wrapper for Parse that omits the error.
42
// In case of an error, a sensible default is returned.
43
func Make(s string) Tag {
44
	return Default.Make(s)
45
}
46

47
// Make is a convenience wrapper for c.Parse that omits the error.
48
// In case of an error, a sensible default is returned.
49
func (c CanonType) Make(s string) Tag {
50
	t, _ := c.Parse(s)
51
	return t
52
}
53

54
// Raw returns the raw base language, script and region, without making an
55
// attempt to infer their values.
56
func (t Tag) Raw() (b Base, s Script, r Region) {
57
	tt := t.tag()
58
	return Base{tt.LangID}, Script{tt.ScriptID}, Region{tt.RegionID}
59
}
60

61
// IsRoot returns true if t is equal to language "und".
62
func (t Tag) IsRoot() bool {
63
	return compact.Tag(t).IsRoot()
64
}
65

66
// CanonType can be used to enable or disable various types of canonicalization.
67
type CanonType int
68

69
const (
70
	// Replace deprecated base languages with their preferred replacements.
71
	DeprecatedBase CanonType = 1 << iota
72
	// Replace deprecated scripts with their preferred replacements.
73
	DeprecatedScript
74
	// Replace deprecated regions with their preferred replacements.
75
	DeprecatedRegion
76
	// Remove redundant scripts.
77
	SuppressScript
78
	// Normalize legacy encodings. This includes legacy languages defined in
79
	// CLDR as well as bibliographic codes defined in ISO-639.
80
	Legacy
81
	// Map the dominant language of a macro language group to the macro language
82
	// subtag. For example cmn -> zh.
83
	Macro
84
	// The CLDR flag should be used if full compatibility with CLDR is required.
85
	// There are a few cases where language.Tag may differ from CLDR. To follow all
86
	// of CLDR's suggestions, use All|CLDR.
87
	CLDR
88

89
	// Raw can be used to Compose or Parse without Canonicalization.
90
	Raw CanonType = 0
91

92
	// Replace all deprecated tags with their preferred replacements.
93
	Deprecated = DeprecatedBase | DeprecatedScript | DeprecatedRegion
94

95
	// All canonicalizations recommended by BCP 47.
96
	BCP47 = Deprecated | SuppressScript
97

98
	// All canonicalizations.
99
	All = BCP47 | Legacy | Macro
100

101
	// Default is the canonicalization used by Parse, Make and Compose. To
102
	// preserve as much information as possible, canonicalizations that remove
103
	// potentially valuable information are not included. The Matcher is
104
	// designed to recognize similar tags that would be the same if
105
	// they were canonicalized using All.
106
	Default = Deprecated | Legacy
107

108
	canonLang = DeprecatedBase | Legacy | Macro
109

110
	// TODO: LikelyScript, LikelyRegion: suppress similar to ICU.
111
)
112

113
// canonicalize returns the canonicalized equivalent of the tag and
114
// whether there was any change.
115
func canonicalize(c CanonType, t language.Tag) (language.Tag, bool) {
116
	if c == Raw {
117
		return t, false
118
	}
119
	changed := false
120
	if c&SuppressScript != 0 {
121
		if t.LangID.SuppressScript() == t.ScriptID {
122
			t.ScriptID = 0
123
			changed = true
124
		}
125
	}
126
	if c&canonLang != 0 {
127
		for {
128
			if l, aliasType := t.LangID.Canonicalize(); l != t.LangID {
129
				switch aliasType {
130
				case language.Legacy:
131
					if c&Legacy != 0 {
132
						if t.LangID == _sh && t.ScriptID == 0 {
133
							t.ScriptID = _Latn
134
						}
135
						t.LangID = l
136
						changed = true
137
					}
138
				case language.Macro:
139
					if c&Macro != 0 {
140
						// We deviate here from CLDR. The mapping "nb" -> "no"
141
						// qualifies as a typical Macro language mapping.  However,
142
						// for legacy reasons, CLDR maps "no", the macro language
143
						// code for Norwegian, to the dominant variant "nb". This
144
						// change is currently under consideration for CLDR as well.
145
						// See https://unicode.org/cldr/trac/ticket/2698 and also
146
						// https://unicode.org/cldr/trac/ticket/1790 for some of the
147
						// practical implications. TODO: this check could be removed
148
						// if CLDR adopts this change.
149
						if c&CLDR == 0 || t.LangID != _nb {
150
							changed = true
151
							t.LangID = l
152
						}
153
					}
154
				case language.Deprecated:
155
					if c&DeprecatedBase != 0 {
156
						if t.LangID == _mo && t.RegionID == 0 {
157
							t.RegionID = _MD
158
						}
159
						t.LangID = l
160
						changed = true
161
						// Other canonicalization types may still apply.
162
						continue
163
					}
164
				}
165
			} else if c&Legacy != 0 && t.LangID == _no && c&CLDR != 0 {
166
				t.LangID = _nb
167
				changed = true
168
			}
169
			break
170
		}
171
	}
172
	if c&DeprecatedScript != 0 {
173
		if t.ScriptID == _Qaai {
174
			changed = true
175
			t.ScriptID = _Zinh
176
		}
177
	}
178
	if c&DeprecatedRegion != 0 {
179
		if r := t.RegionID.Canonicalize(); r != t.RegionID {
180
			changed = true
181
			t.RegionID = r
182
		}
183
	}
184
	return t, changed
185
}
186

187
// Canonicalize returns the canonicalized equivalent of the tag.
188
func (c CanonType) Canonicalize(t Tag) (Tag, error) {
189
	// First try fast path.
190
	if t.isCompact() {
191
		if _, changed := canonicalize(c, compact.Tag(t).Tag()); !changed {
192
			return t, nil
193
		}
194
	}
195
	// It is unlikely that one will canonicalize a tag after matching. So do
196
	// a slow but simple approach here.
197
	if tag, changed := canonicalize(c, t.tag()); changed {
198
		tag.RemakeString()
199
		return makeTag(tag), nil
200
	}
201
	return t, nil
202

203
}
204

205
// Confidence indicates the level of certainty for a given return value.
206
// For example, Serbian may be written in Cyrillic or Latin script.
207
// The confidence level indicates whether a value was explicitly specified,
208
// whether it is typically the only possible value, or whether there is
209
// an ambiguity.
210
type Confidence int
211

212
const (
213
	No    Confidence = iota // full confidence that there was no match
214
	Low                     // most likely value picked out of a set of alternatives
215
	High                    // value is generally assumed to be the correct match
216
	Exact                   // exact match or explicitly specified value
217
)
218

219
var confName = []string{"No", "Low", "High", "Exact"}
220

221
func (c Confidence) String() string {
222
	return confName[c]
223
}
224

225
// String returns the canonical string representation of the language tag.
226
func (t Tag) String() string {
227
	return t.tag().String()
228
}
229

230
// MarshalText implements encoding.TextMarshaler.
231
func (t Tag) MarshalText() (text []byte, err error) {
232
	return t.tag().MarshalText()
233
}
234

235
// UnmarshalText implements encoding.TextUnmarshaler.
236
func (t *Tag) UnmarshalText(text []byte) error {
237
	var tag language.Tag
238
	err := tag.UnmarshalText(text)
239
	*t = makeTag(tag)
240
	return err
241
}
242

243
// Base returns the base language of the language tag. If the base language is
244
// unspecified, an attempt will be made to infer it from the context.
245
// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
246
func (t Tag) Base() (Base, Confidence) {
247
	if b := t.lang(); b != 0 {
248
		return Base{b}, Exact
249
	}
250
	tt := t.tag()
251
	c := High
252
	if tt.ScriptID == 0 && !tt.RegionID.IsCountry() {
253
		c = Low
254
	}
255
	if tag, err := tt.Maximize(); err == nil && tag.LangID != 0 {
256
		return Base{tag.LangID}, c
257
	}
258
	return Base{0}, No
259
}
260

261
// Script infers the script for the language tag. If it was not explicitly given, it will infer
262
// a most likely candidate.
263
// If more than one script is commonly used for a language, the most likely one
264
// is returned with a low confidence indication. For example, it returns (Cyrl, Low)
265
// for Serbian.
266
// If a script cannot be inferred (Zzzz, No) is returned. We do not use Zyyy (undetermined)
267
// as one would suspect from the IANA registry for BCP 47. In a Unicode context Zyyy marks
268
// common characters (like 1, 2, 3, '.', etc.) and is therefore more like multiple scripts.
269
// See https://www.unicode.org/reports/tr24/#Values for more details. Zzzz is also used for
270
// unknown value in CLDR.  (Zzzz, Exact) is returned if Zzzz was explicitly specified.
271
// Note that an inferred script is never guaranteed to be the correct one. Latin is
272
// almost exclusively used for Afrikaans, but Arabic has been used for some texts
273
// in the past.  Also, the script that is commonly used may change over time.
274
// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
275
func (t Tag) Script() (Script, Confidence) {
276
	if scr := t.script(); scr != 0 {
277
		return Script{scr}, Exact
278
	}
279
	tt := t.tag()
280
	sc, c := language.Script(_Zzzz), No
281
	if scr := tt.LangID.SuppressScript(); scr != 0 {
282
		// Note: it is not always the case that a language with a suppress
283
		// script value is only written in one script (e.g. kk, ms, pa).
284
		if tt.RegionID == 0 {
285
			return Script{scr}, High
286
		}
287
		sc, c = scr, High
288
	}
289
	if tag, err := tt.Maximize(); err == nil {
290
		if tag.ScriptID != sc {
291
			sc, c = tag.ScriptID, Low
292
		}
293
	} else {
294
		tt, _ = canonicalize(Deprecated|Macro, tt)
295
		if tag, err := tt.Maximize(); err == nil && tag.ScriptID != sc {
296
			sc, c = tag.ScriptID, Low
297
		}
298
	}
299
	return Script{sc}, c
300
}
301

302
// Region returns the region for the language tag. If it was not explicitly given, it will
303
// infer a most likely candidate from the context.
304
// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
305
func (t Tag) Region() (Region, Confidence) {
306
	if r := t.region(); r != 0 {
307
		return Region{r}, Exact
308
	}
309
	tt := t.tag()
310
	if tt, err := tt.Maximize(); err == nil {
311
		return Region{tt.RegionID}, Low // TODO: differentiate between high and low.
312
	}
313
	tt, _ = canonicalize(Deprecated|Macro, tt)
314
	if tag, err := tt.Maximize(); err == nil {
315
		return Region{tag.RegionID}, Low
316
	}
317
	return Region{_ZZ}, No // TODO: return world instead of undetermined?
318
}
319

320
// Variants returns the variants specified explicitly for this language tag.
321
// or nil if no variant was specified.
322
func (t Tag) Variants() []Variant {
323
	if !compact.Tag(t).MayHaveVariants() {
324
		return nil
325
	}
326
	v := []Variant{}
327
	x, str := "", t.tag().Variants()
328
	for str != "" {
329
		x, str = nextToken(str)
330
		v = append(v, Variant{x})
331
	}
332
	return v
333
}
334

335
// Parent returns the CLDR parent of t. In CLDR, missing fields in data for a
336
// specific language are substituted with fields from the parent language.
337
// The parent for a language may change for newer versions of CLDR.
338
//
339
// Parent returns a tag for a less specific language that is mutually
340
// intelligible or Und if there is no such language. This may not be the same as
341
// simply stripping the last BCP 47 subtag. For instance, the parent of "zh-TW"
342
// is "zh-Hant", and the parent of "zh-Hant" is "und".
343
func (t Tag) Parent() Tag {
344
	return Tag(compact.Tag(t).Parent())
345
}
346

347
// nextToken returns token t and the rest of the string.
348
func nextToken(s string) (t, tail string) {
349
	p := strings.Index(s[1:], "-")
350
	if p == -1 {
351
		return s[1:], ""
352
	}
353
	p++
354
	return s[1:p], s[p:]
355
}
356

357
// Extension is a single BCP 47 extension.
358
type Extension struct {
359
	s string
360
}
361

362
// String returns the string representation of the extension, including the
363
// type tag.
364
func (e Extension) String() string {
365
	return e.s
366
}
367

368
// ParseExtension parses s as an extension and returns it on success.
369
func ParseExtension(s string) (e Extension, err error) {
370
	ext, err := language.ParseExtension(s)
371
	return Extension{ext}, err
372
}
373

374
// Type returns the one-byte extension type of e. It returns 0 for the zero
375
// exception.
376
func (e Extension) Type() byte {
377
	if e.s == "" {
378
		return 0
379
	}
380
	return e.s[0]
381
}
382

383
// Tokens returns the list of tokens of e.
384
func (e Extension) Tokens() []string {
385
	return strings.Split(e.s, "-")
386
}
387

388
// Extension returns the extension of type x for tag t. It will return
389
// false for ok if t does not have the requested extension. The returned
390
// extension will be invalid in this case.
391
func (t Tag) Extension(x byte) (ext Extension, ok bool) {
392
	if !compact.Tag(t).MayHaveExtensions() {
393
		return Extension{}, false
394
	}
395
	e, ok := t.tag().Extension(x)
396
	return Extension{e}, ok
397
}
398

399
// Extensions returns all extensions of t.
400
func (t Tag) Extensions() []Extension {
401
	if !compact.Tag(t).MayHaveExtensions() {
402
		return nil
403
	}
404
	e := []Extension{}
405
	for _, ext := range t.tag().Extensions() {
406
		e = append(e, Extension{ext})
407
	}
408
	return e
409
}
410

411
// TypeForKey returns the type associated with the given key, where key and type
412
// are of the allowed values defined for the Unicode locale extension ('u') in
413
// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
414
// TypeForKey will traverse the inheritance chain to get the correct value.
415
//
416
// If there are multiple types associated with a key, only the first will be
417
// returned. If there is no type associated with a key, it returns the empty
418
// string.
419
func (t Tag) TypeForKey(key string) string {
420
	if !compact.Tag(t).MayHaveExtensions() {
421
		if key != "rg" && key != "va" {
422
			return ""
423
		}
424
	}
425
	return t.tag().TypeForKey(key)
426
}
427

428
// SetTypeForKey returns a new Tag with the key set to type, where key and type
429
// are of the allowed values defined for the Unicode locale extension ('u') in
430
// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
431
// An empty value removes an existing pair with the same key.
432
func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
433
	tt, err := t.tag().SetTypeForKey(key, value)
434
	return makeTag(tt), err
435
}
436

437
// NumCompactTags is the number of compact tags. The maximum tag is
438
// NumCompactTags-1.
439
const NumCompactTags = compact.NumCompactTags
440

441
// CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags
442
// for which data exists in the text repository.The index will change over time
443
// and should not be stored in persistent storage. If t does not match a compact
444
// index, exact will be false and the compact index will be returned for the
445
// first match after repeatedly taking the Parent of t.
446
func CompactIndex(t Tag) (index int, exact bool) {
447
	id, exact := compact.LanguageID(compact.Tag(t))
448
	return int(id), exact
449
}
450

451
var root = language.Tag{}
452

453
// Base is an ISO 639 language code, used for encoding the base language
454
// of a language tag.
455
type Base struct {
456
	langID language.Language
457
}
458

459
// ParseBase parses a 2- or 3-letter ISO 639 code.
460
// It returns a ValueError if s is a well-formed but unknown language identifier
461
// or another error if another error occurred.
462
func ParseBase(s string) (Base, error) {
463
	l, err := language.ParseBase(s)
464
	return Base{l}, err
465
}
466

467
// String returns the BCP 47 representation of the base language.
468
func (b Base) String() string {
469
	return b.langID.String()
470
}
471

472
// ISO3 returns the ISO 639-3 language code.
473
func (b Base) ISO3() string {
474
	return b.langID.ISO3()
475
}
476

477
// IsPrivateUse reports whether this language code is reserved for private use.
478
func (b Base) IsPrivateUse() bool {
479
	return b.langID.IsPrivateUse()
480
}
481

482
// Script is a 4-letter ISO 15924 code for representing scripts.
483
// It is idiomatically represented in title case.
484
type Script struct {
485
	scriptID language.Script
486
}
487

488
// ParseScript parses a 4-letter ISO 15924 code.
489
// It returns a ValueError if s is a well-formed but unknown script identifier
490
// or another error if another error occurred.
491
func ParseScript(s string) (Script, error) {
492
	sc, err := language.ParseScript(s)
493
	return Script{sc}, err
494
}
495

496
// String returns the script code in title case.
497
// It returns "Zzzz" for an unspecified script.
498
func (s Script) String() string {
499
	return s.scriptID.String()
500
}
501

502
// IsPrivateUse reports whether this script code is reserved for private use.
503
func (s Script) IsPrivateUse() bool {
504
	return s.scriptID.IsPrivateUse()
505
}
506

507
// Region is an ISO 3166-1 or UN M.49 code for representing countries and regions.
508
type Region struct {
509
	regionID language.Region
510
}
511

512
// EncodeM49 returns the Region for the given UN M.49 code.
513
// It returns an error if r is not a valid code.
514
func EncodeM49(r int) (Region, error) {
515
	rid, err := language.EncodeM49(r)
516
	return Region{rid}, err
517
}
518

519
// ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code.
520
// It returns a ValueError if s is a well-formed but unknown region identifier
521
// or another error if another error occurred.
522
func ParseRegion(s string) (Region, error) {
523
	r, err := language.ParseRegion(s)
524
	return Region{r}, err
525
}
526

527
// String returns the BCP 47 representation for the region.
528
// It returns "ZZ" for an unspecified region.
529
func (r Region) String() string {
530
	return r.regionID.String()
531
}
532

533
// ISO3 returns the 3-letter ISO code of r.
534
// Note that not all regions have a 3-letter ISO code.
535
// In such cases this method returns "ZZZ".
536
func (r Region) ISO3() string {
537
	return r.regionID.ISO3()
538
}
539

540
// M49 returns the UN M.49 encoding of r, or 0 if this encoding
541
// is not defined for r.
542
func (r Region) M49() int {
543
	return r.regionID.M49()
544
}
545

546
// IsPrivateUse reports whether r has the ISO 3166 User-assigned status. This
547
// may include private-use tags that are assigned by CLDR and used in this
548
// implementation. So IsPrivateUse and IsCountry can be simultaneously true.
549
func (r Region) IsPrivateUse() bool {
550
	return r.regionID.IsPrivateUse()
551
}
552

553
// IsCountry returns whether this region is a country or autonomous area. This
554
// includes non-standard definitions from CLDR.
555
func (r Region) IsCountry() bool {
556
	return r.regionID.IsCountry()
557
}
558

559
// IsGroup returns whether this region defines a collection of regions. This
560
// includes non-standard definitions from CLDR.
561
func (r Region) IsGroup() bool {
562
	return r.regionID.IsGroup()
563
}
564

565
// Contains returns whether Region c is contained by Region r. It returns true
566
// if c == r.
567
func (r Region) Contains(c Region) bool {
568
	return r.regionID.Contains(c.regionID)
569
}
570

571
// TLD returns the country code top-level domain (ccTLD). UK is returned for GB.
572
// In all other cases it returns either the region itself or an error.
573
//
574
// This method may return an error for a region for which there exists a
575
// canonical form with a ccTLD. To get that ccTLD canonicalize r first. The
576
// region will already be canonicalized it was obtained from a Tag that was
577
// obtained using any of the default methods.
578
func (r Region) TLD() (Region, error) {
579
	tld, err := r.regionID.TLD()
580
	return Region{tld}, err
581
}
582

583
// Canonicalize returns the region or a possible replacement if the region is
584
// deprecated. It will not return a replacement for deprecated regions that
585
// are split into multiple regions.
586
func (r Region) Canonicalize() Region {
587
	return Region{r.regionID.Canonicalize()}
588
}
589

590
// Variant represents a registered variant of a language as defined by BCP 47.
591
type Variant struct {
592
	variant string
593
}
594

595
// ParseVariant parses and returns a Variant. An error is returned if s is not
596
// a valid variant.
597
func ParseVariant(s string) (Variant, error) {
598
	v, err := language.ParseVariant(s)
599
	return Variant{v.String()}, err
600
}
601

602
// String returns the string representation of the variant.
603
func (v Variant) String() string {
604
	return v.variant
605
}
606

607