1
// Copyright 2013 The Go Authors. All rights reserved.
2
// Use of this source code is governed by a BSD-style
3
// license that can be found in the LICENSE file.
4

5
//go:generate go run gen.go gen_common.go -output tables.go
6

7
package language // import "golang.org/x/text/internal/language"
8

9
// TODO: Remove above NOTE after:
10
// - verifying that tables are dropped correctly (most notably matcher tables).
11

12
import (
13
	"errors"
14
	"fmt"
15
	"strings"
16
)
17

18
const (
19
	// maxCoreSize is the maximum size of a BCP 47 tag without variants and
20
	// extensions. Equals max lang (3) + script (4) + max reg (3) + 2 dashes.
21
	maxCoreSize = 12
22

23
	// max99thPercentileSize is a somewhat arbitrary buffer size that presumably
24
	// is large enough to hold at least 99% of the BCP 47 tags.
25
	max99thPercentileSize = 32
26

27
	// maxSimpleUExtensionSize is the maximum size of a -u extension with one
28
	// key-type pair. Equals len("-u-") + key (2) + dash + max value (8).
29
	maxSimpleUExtensionSize = 14
30
)
31

32
// Tag represents a BCP 47 language tag. It is used to specify an instance of a
33
// specific language or locale. All language tag values are guaranteed to be
34
// well-formed. The zero value of Tag is Und.
35
type Tag struct {
36
	// TODO: the following fields have the form TagTypeID. This name is chosen
37
	// to allow refactoring the public package without conflicting with its
38
	// Base, Script, and Region methods. Once the transition is fully completed
39
	// the ID can be stripped from the name.
40

41
	LangID   Language
42
	RegionID Region
43
	// TODO: we will soon run out of positions for ScriptID. Idea: instead of
44
	// storing lang, region, and ScriptID codes, store only the compact index and
45
	// have a lookup table from this code to its expansion. This greatly speeds
46
	// up table lookup, speed up common variant cases.
47
	// This will also immediately free up 3 extra bytes. Also, the pVariant
48
	// field can now be moved to the lookup table, as the compact index uniquely
49
	// determines the offset of a possible variant.
50
	ScriptID Script
51
	pVariant byte   // offset in str, includes preceding '-'
52
	pExt     uint16 // offset of first extension, includes preceding '-'
53

54
	// str is the string representation of the Tag. It will only be used if the
55
	// tag has variants or extensions.
56
	str string
57
}
58

59
// Make is a convenience wrapper for Parse that omits the error.
60
// In case of an error, a sensible default is returned.
61
func Make(s string) Tag {
62
	t, _ := Parse(s)
63
	return t
64
}
65

66
// Raw returns the raw base language, script and region, without making an
67
// attempt to infer their values.
68
// TODO: consider removing
69
func (t Tag) Raw() (b Language, s Script, r Region) {
70
	return t.LangID, t.ScriptID, t.RegionID
71
}
72

73
// equalTags compares language, script and region subtags only.
74
func (t Tag) equalTags(a Tag) bool {
75
	return t.LangID == a.LangID && t.ScriptID == a.ScriptID && t.RegionID == a.RegionID
76
}
77

78
// IsRoot returns true if t is equal to language "und".
79
func (t Tag) IsRoot() bool {
80
	if int(t.pVariant) < len(t.str) {
81
		return false
82
	}
83
	return t.equalTags(Und)
84
}
85

86
// IsPrivateUse reports whether the Tag consists solely of an IsPrivateUse use
87
// tag.
88
func (t Tag) IsPrivateUse() bool {
89
	return t.str != "" && t.pVariant == 0
90
}
91

92
// RemakeString is used to update t.str in case lang, script or region changed.
93
// It is assumed that pExt and pVariant still point to the start of the
94
// respective parts.
95
func (t *Tag) RemakeString() {
96
	if t.str == "" {
97
		return
98
	}
99
	extra := t.str[t.pVariant:]
100
	if t.pVariant > 0 {
101
		extra = extra[1:]
102
	}
103
	if t.equalTags(Und) && strings.HasPrefix(extra, "x-") {
104
		t.str = extra
105
		t.pVariant = 0
106
		t.pExt = 0
107
		return
108
	}
109
	var buf [max99thPercentileSize]byte // avoid extra memory allocation in most cases.
110
	b := buf[:t.genCoreBytes(buf[:])]
111
	if extra != "" {
112
		diff := len(b) - int(t.pVariant)
113
		b = append(b, '-')
114
		b = append(b, extra...)
115
		t.pVariant = uint8(int(t.pVariant) + diff)
116
		t.pExt = uint16(int(t.pExt) + diff)
117
	} else {
118
		t.pVariant = uint8(len(b))
119
		t.pExt = uint16(len(b))
120
	}
121
	t.str = string(b)
122
}
123

124
// genCoreBytes writes a string for the base languages, script and region tags
125
// to the given buffer and returns the number of bytes written. It will never
126
// write more than maxCoreSize bytes.
127
func (t *Tag) genCoreBytes(buf []byte) int {
128
	n := t.LangID.StringToBuf(buf[:])
129
	if t.ScriptID != 0 {
130
		n += copy(buf[n:], "-")
131
		n += copy(buf[n:], t.ScriptID.String())
132
	}
133
	if t.RegionID != 0 {
134
		n += copy(buf[n:], "-")
135
		n += copy(buf[n:], t.RegionID.String())
136
	}
137
	return n
138
}
139

140
// String returns the canonical string representation of the language tag.
141
func (t Tag) String() string {
142
	if t.str != "" {
143
		return t.str
144
	}
145
	if t.ScriptID == 0 && t.RegionID == 0 {
146
		return t.LangID.String()
147
	}
148
	buf := [maxCoreSize]byte{}
149
	return string(buf[:t.genCoreBytes(buf[:])])
150
}
151

152
// MarshalText implements encoding.TextMarshaler.
153
func (t Tag) MarshalText() (text []byte, err error) {
154
	if t.str != "" {
155
		text = append(text, t.str...)
156
	} else if t.ScriptID == 0 && t.RegionID == 0 {
157
		text = append(text, t.LangID.String()...)
158
	} else {
159
		buf := [maxCoreSize]byte{}
160
		text = buf[:t.genCoreBytes(buf[:])]
161
	}
162
	return text, nil
163
}
164

165
// UnmarshalText implements encoding.TextUnmarshaler.
166
func (t *Tag) UnmarshalText(text []byte) error {
167
	tag, err := Parse(string(text))
168
	*t = tag
169
	return err
170
}
171

172
// Variants returns the part of the tag holding all variants or the empty string
173
// if there are no variants defined.
174
func (t Tag) Variants() string {
175
	if t.pVariant == 0 {
176
		return ""
177
	}
178
	return t.str[t.pVariant:t.pExt]
179
}
180

181
// VariantOrPrivateUseTags returns variants or private use tags.
182
func (t Tag) VariantOrPrivateUseTags() string {
183
	if t.pExt > 0 {
184
		return t.str[t.pVariant:t.pExt]
185
	}
186
	return t.str[t.pVariant:]
187
}
188

189
// HasString reports whether this tag defines more than just the raw
190
// components.
191
func (t Tag) HasString() bool {
192
	return t.str != ""
193
}
194

195
// Parent returns the CLDR parent of t. In CLDR, missing fields in data for a
196
// specific language are substituted with fields from the parent language.
197
// The parent for a language may change for newer versions of CLDR.
198
func (t Tag) Parent() Tag {
199
	if t.str != "" {
200
		// Strip the variants and extensions.
201
		b, s, r := t.Raw()
202
		t = Tag{LangID: b, ScriptID: s, RegionID: r}
203
		if t.RegionID == 0 && t.ScriptID != 0 && t.LangID != 0 {
204
			base, _ := addTags(Tag{LangID: t.LangID})
205
			if base.ScriptID == t.ScriptID {
206
				return Tag{LangID: t.LangID}
207
			}
208
		}
209
		return t
210
	}
211
	if t.LangID != 0 {
212
		if t.RegionID != 0 {
213
			maxScript := t.ScriptID
214
			if maxScript == 0 {
215
				max, _ := addTags(t)
216
				maxScript = max.ScriptID
217
			}
218

219
			for i := range parents {
220
				if Language(parents[i].lang) == t.LangID && Script(parents[i].maxScript) == maxScript {
221
					for _, r := range parents[i].fromRegion {
222
						if Region(r) == t.RegionID {
223
							return Tag{
224
								LangID:   t.LangID,
225
								ScriptID: Script(parents[i].script),
226
								RegionID: Region(parents[i].toRegion),
227
							}
228
						}
229
					}
230
				}
231
			}
232

233
			// Strip the script if it is the default one.
234
			base, _ := addTags(Tag{LangID: t.LangID})
235
			if base.ScriptID != maxScript {
236
				return Tag{LangID: t.LangID, ScriptID: maxScript}
237
			}
238
			return Tag{LangID: t.LangID}
239
		} else if t.ScriptID != 0 {
240
			// The parent for an base-script pair with a non-default script is
241
			// "und" instead of the base language.
242
			base, _ := addTags(Tag{LangID: t.LangID})
243
			if base.ScriptID != t.ScriptID {
244
				return Und
245
			}
246
			return Tag{LangID: t.LangID}
247
		}
248
	}
249
	return Und
250
}
251

252
// ParseExtension parses s as an extension and returns it on success.
253
func ParseExtension(s string) (ext string, err error) {
254
	defer func() {
255
		if recover() != nil {
256
			ext = ""
257
			err = ErrSyntax
258
		}
259
	}()
260

261
	scan := makeScannerString(s)
262
	var end int
263
	if n := len(scan.token); n != 1 {
264
		return "", ErrSyntax
265
	}
266
	scan.toLower(0, len(scan.b))
267
	end = parseExtension(&scan)
268
	if end != len(s) {
269
		return "", ErrSyntax
270
	}
271
	return string(scan.b), nil
272
}
273

274
// HasVariants reports whether t has variants.
275
func (t Tag) HasVariants() bool {
276
	return uint16(t.pVariant) < t.pExt
277
}
278

279
// HasExtensions reports whether t has extensions.
280
func (t Tag) HasExtensions() bool {
281
	return int(t.pExt) < len(t.str)
282
}
283

284
// Extension returns the extension of type x for tag t. It will return
285
// false for ok if t does not have the requested extension. The returned
286
// extension will be invalid in this case.
287
func (t Tag) Extension(x byte) (ext string, ok bool) {
288
	for i := int(t.pExt); i < len(t.str)-1; {
289
		var ext string
290
		i, ext = getExtension(t.str, i)
291
		if ext[0] == x {
292
			return ext, true
293
		}
294
	}
295
	return "", false
296
}
297

298
// Extensions returns all extensions of t.
299
func (t Tag) Extensions() []string {
300
	e := []string{}
301
	for i := int(t.pExt); i < len(t.str)-1; {
302
		var ext string
303
		i, ext = getExtension(t.str, i)
304
		e = append(e, ext)
305
	}
306
	return e
307
}
308

309
// TypeForKey returns the type associated with the given key, where key and type
310
// are of the allowed values defined for the Unicode locale extension ('u') in
311
// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
312
// TypeForKey will traverse the inheritance chain to get the correct value.
313
//
314
// If there are multiple types associated with a key, only the first will be
315
// returned. If there is no type associated with a key, it returns the empty
316
// string.
317
func (t Tag) TypeForKey(key string) string {
318
	if _, start, end, _ := t.findTypeForKey(key); end != start {
319
		s := t.str[start:end]
320
		if p := strings.IndexByte(s, '-'); p >= 0 {
321
			s = s[:p]
322
		}
323
		return s
324
	}
325
	return ""
326
}
327

328
var (
329
	errPrivateUse       = errors.New("cannot set a key on a private use tag")
330
	errInvalidArguments = errors.New("invalid key or type")
331
)
332

333
// SetTypeForKey returns a new Tag with the key set to type, where key and type
334
// are of the allowed values defined for the Unicode locale extension ('u') in
335
// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
336
// An empty value removes an existing pair with the same key.
337
func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
338
	if t.IsPrivateUse() {
339
		return t, errPrivateUse
340
	}
341
	if len(key) != 2 {
342
		return t, errInvalidArguments
343
	}
344

345
	// Remove the setting if value is "".
346
	if value == "" {
347
		start, sep, end, _ := t.findTypeForKey(key)
348
		if start != sep {
349
			// Remove a possible empty extension.
350
			switch {
351
			case t.str[start-2] != '-': // has previous elements.
352
			case end == len(t.str), // end of string
353
				end+2 < len(t.str) && t.str[end+2] == '-': // end of extension
354
				start -= 2
355
			}
356
			if start == int(t.pVariant) && end == len(t.str) {
357
				t.str = ""
358
				t.pVariant, t.pExt = 0, 0
359
			} else {
360
				t.str = fmt.Sprintf("%s%s", t.str[:start], t.str[end:])
361
			}
362
		}
363
		return t, nil
364
	}
365

366
	if len(value) < 3 || len(value) > 8 {
367
		return t, errInvalidArguments
368
	}
369

370
	var (
371
		buf    [maxCoreSize + maxSimpleUExtensionSize]byte
372
		uStart int // start of the -u extension.
373
	)
374

375
	// Generate the tag string if needed.
376
	if t.str == "" {
377
		uStart = t.genCoreBytes(buf[:])
378
		buf[uStart] = '-'
379
		uStart++
380
	}
381

382
	// Create new key-type pair and parse it to verify.
383
	b := buf[uStart:]
384
	copy(b, "u-")
385
	copy(b[2:], key)
386
	b[4] = '-'
387
	b = b[:5+copy(b[5:], value)]
388
	scan := makeScanner(b)
389
	if parseExtensions(&scan); scan.err != nil {
390
		return t, scan.err
391
	}
392

393
	// Assemble the replacement string.
394
	if t.str == "" {
395
		t.pVariant, t.pExt = byte(uStart-1), uint16(uStart-1)
396
		t.str = string(buf[:uStart+len(b)])
397
	} else {
398
		s := t.str
399
		start, sep, end, hasExt := t.findTypeForKey(key)
400
		if start == sep {
401
			if hasExt {
402
				b = b[2:]
403
			}
404
			t.str = fmt.Sprintf("%s-%s%s", s[:sep], b, s[end:])
405
		} else {
406
			t.str = fmt.Sprintf("%s-%s%s", s[:start+3], value, s[end:])
407
		}
408
	}
409
	return t, nil
410
}
411

412
// findTypeForKey returns the start and end position for the type corresponding
413
// to key or the point at which to insert the key-value pair if the type
414
// wasn't found. The hasExt return value reports whether an -u extension was present.
415
// Note: the extensions are typically very small and are likely to contain
416
// only one key-type pair.
417
func (t Tag) findTypeForKey(key string) (start, sep, end int, hasExt bool) {
418
	p := int(t.pExt)
419
	if len(key) != 2 || p == len(t.str) || p == 0 {
420
		return p, p, p, false
421
	}
422
	s := t.str
423

424
	// Find the correct extension.
425
	for p++; s[p] != 'u'; p++ {
426
		if s[p] > 'u' {
427
			p--
428
			return p, p, p, false
429
		}
430
		if p = nextExtension(s, p); p == len(s) {
431
			return len(s), len(s), len(s), false
432
		}
433
	}
434
	// Proceed to the hyphen following the extension name.
435
	p++
436

437
	// curKey is the key currently being processed.
438
	curKey := ""
439

440
	// Iterate over keys until we get the end of a section.
441
	for {
442
		end = p
443
		for p++; p < len(s) && s[p] != '-'; p++ {
444
		}
445
		n := p - end - 1
446
		if n <= 2 && curKey == key {
447
			if sep < end {
448
				sep++
449
			}
450
			return start, sep, end, true
451
		}
452
		switch n {
453
		case 0, // invalid string
454
			1: // next extension
455
			return end, end, end, true
456
		case 2:
457
			// next key
458
			curKey = s[end+1 : p]
459
			if curKey > key {
460
				return end, end, end, true
461
			}
462
			start = end
463
			sep = p
464
		}
465
	}
466
}
467

468
// ParseBase parses a 2- or 3-letter ISO 639 code.
469
// It returns a ValueError if s is a well-formed but unknown language identifier
470
// or another error if another error occurred.
471
func ParseBase(s string) (l Language, err error) {
472
	defer func() {
473
		if recover() != nil {
474
			l = 0
475
			err = ErrSyntax
476
		}
477
	}()
478

479
	if n := len(s); n < 2 || 3 < n {
480
		return 0, ErrSyntax
481
	}
482
	var buf [3]byte
483
	return getLangID(buf[:copy(buf[:], s)])
484
}
485

486
// ParseScript parses a 4-letter ISO 15924 code.
487
// It returns a ValueError if s is a well-formed but unknown script identifier
488
// or another error if another error occurred.
489
func ParseScript(s string) (scr Script, err error) {
490
	defer func() {
491
		if recover() != nil {
492
			scr = 0
493
			err = ErrSyntax
494
		}
495
	}()
496

497
	if len(s) != 4 {
498
		return 0, ErrSyntax
499
	}
500
	var buf [4]byte
501
	return getScriptID(script, buf[:copy(buf[:], s)])
502
}
503

504
// EncodeM49 returns the Region for the given UN M.49 code.
505
// It returns an error if r is not a valid code.
506
func EncodeM49(r int) (Region, error) {
507
	return getRegionM49(r)
508
}
509

510
// ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code.
511
// It returns a ValueError if s is a well-formed but unknown region identifier
512
// or another error if another error occurred.
513
func ParseRegion(s string) (r Region, err error) {
514
	defer func() {
515
		if recover() != nil {
516
			r = 0
517
			err = ErrSyntax
518
		}
519
	}()
520

521
	if n := len(s); n < 2 || 3 < n {
522
		return 0, ErrSyntax
523
	}
524
	var buf [3]byte
525
	return getRegionID(buf[:copy(buf[:], s)])
526
}
527

528
// IsCountry returns whether this region is a country or autonomous area. This
529
// includes non-standard definitions from CLDR.
530
func (r Region) IsCountry() bool {
531
	if r == 0 || r.IsGroup() || r.IsPrivateUse() && r != _XK {
532
		return false
533
	}
534
	return true
535
}
536

537
// IsGroup returns whether this region defines a collection of regions. This
538
// includes non-standard definitions from CLDR.
539
func (r Region) IsGroup() bool {
540
	if r == 0 {
541
		return false
542
	}
543
	return int(regionInclusion[r]) < len(regionContainment)
544
}
545

546
// Contains returns whether Region c is contained by Region r. It returns true
547
// if c == r.
548
func (r Region) Contains(c Region) bool {
549
	if r == c {
550
		return true
551
	}
552
	g := regionInclusion[r]
553
	if g >= nRegionGroups {
554
		return false
555
	}
556
	m := regionContainment[g]
557

558
	d := regionInclusion[c]
559
	b := regionInclusionBits[d]
560

561
	// A contained country may belong to multiple disjoint groups. Matching any
562
	// of these indicates containment. If the contained region is a group, it
563
	// must strictly be a subset.
564
	if d >= nRegionGroups {
565
		return b&m != 0
566
	}
567
	return b&^m == 0
568
}
569

570
var errNoTLD = errors.New("language: region is not a valid ccTLD")
571

572
// TLD returns the country code top-level domain (ccTLD). UK is returned for GB.
573
// In all other cases it returns either the region itself or an error.
574
//
575
// This method may return an error for a region for which there exists a
576
// canonical form with a ccTLD. To get that ccTLD canonicalize r first. The
577
// region will already be canonicalized it was obtained from a Tag that was
578
// obtained using any of the default methods.
579
func (r Region) TLD() (Region, error) {
580
	// See http://en.wikipedia.org/wiki/Country_code_top-level_domain for the
581
	// difference between ISO 3166-1 and IANA ccTLD.
582
	if r == _GB {
583
		r = _UK
584
	}
585
	if (r.typ() & ccTLD) == 0 {
586
		return 0, errNoTLD
587
	}
588
	return r, nil
589
}
590

591
// Canonicalize returns the region or a possible replacement if the region is
592
// deprecated. It will not return a replacement for deprecated regions that
593
// are split into multiple regions.
594
func (r Region) Canonicalize() Region {
595
	if cr := normRegion(r); cr != 0 {
596
		return cr
597
	}
598
	return r
599
}
600

601
// Variant represents a registered variant of a language as defined by BCP 47.
602
type Variant struct {
603
	ID  uint8
604
	str string
605
}
606

607
// ParseVariant parses and returns a Variant. An error is returned if s is not
608
// a valid variant.
609
func ParseVariant(s string) (v Variant, err error) {
610
	defer func() {
611
		if recover() != nil {
612
			v = Variant{}
613
			err = ErrSyntax
614
		}
615
	}()
616

617
	s = strings.ToLower(s)
618
	if id, ok := variantIndex[s]; ok {
619
		return Variant{id, s}, nil
620
	}
621
	return Variant{}, NewValueError([]byte(s))
622
}
623

624
// String returns the string representation of the variant.
625
func (v Variant) String() string {
626
	return v.str
627
}
628

629