1
// SPDX-License-Identifier: GPL-2.0
2
/*
3
 * Basic Node interface support
4
 */
5

6
#include <linux/module.h>
7
#include <linux/init.h>
8
#include <linux/mm.h>
9
#include <linux/memory.h>
10
#include <linux/mempolicy.h>
11
#include <linux/vmstat.h>
12
#include <linux/notifier.h>
13
#include <linux/node.h>
14
#include <linux/hugetlb.h>
15
#include <linux/compaction.h>
16
#include <linux/cpumask.h>
17
#include <linux/topology.h>
18
#include <linux/nodemask.h>
19
#include <linux/cpu.h>
20
#include <linux/device.h>
21
#include <linux/pm_runtime.h>
22
#include <linux/swap.h>
23
#include <linux/slab.h>
24
#include <linux/memblock.h>
25

26
static const struct bus_type node_subsys = {
27
	.name = "node",
28
	.dev_name = "node",
29
};
30

31
static inline ssize_t cpumap_read(struct file *file, struct kobject *kobj,
32
				  const struct bin_attribute *attr, char *buf,
33
				  loff_t off, size_t count)
34
{
35
	struct device *dev = kobj_to_dev(kobj);
36
	struct node *node_dev = to_node(dev);
37
	cpumask_var_t mask;
38
	ssize_t n;
39

40
	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
41
		return 0;
42

43
	cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask);
44
	n = cpumap_print_bitmask_to_buf(buf, mask, off, count);
45
	free_cpumask_var(mask);
46

47
	return n;
48
}
49

50
static const BIN_ATTR_RO(cpumap, CPUMAP_FILE_MAX_BYTES);
51

52
static inline ssize_t cpulist_read(struct file *file, struct kobject *kobj,
53
				   const struct bin_attribute *attr, char *buf,
54
				   loff_t off, size_t count)
55
{
56
	struct device *dev = kobj_to_dev(kobj);
57
	struct node *node_dev = to_node(dev);
58
	cpumask_var_t mask;
59
	ssize_t n;
60

61
	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
62
		return 0;
63

64
	cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask);
65
	n = cpumap_print_list_to_buf(buf, mask, off, count);
66
	free_cpumask_var(mask);
67

68
	return n;
69
}
70

71
static const BIN_ATTR_RO(cpulist, CPULIST_FILE_MAX_BYTES);
72

73
/**
74
 * struct node_access_nodes - Access class device to hold user visible
75
 * 			      relationships to other nodes.
76
 * @dev:	Device for this memory access class
77
 * @list_node:	List element in the node's access list
78
 * @access:	The access class rank
79
 * @coord:	Heterogeneous memory performance coordinates
80
 */
81
struct node_access_nodes {
82
	struct device		dev;
83
	struct list_head	list_node;
84
	unsigned int		access;
85
#ifdef CONFIG_HMEM_REPORTING
86
	struct access_coordinate	coord;
87
#endif
88
};
89
#define to_access_nodes(dev) container_of(dev, struct node_access_nodes, dev)
90

91
static struct attribute *node_init_access_node_attrs[] = {
92
	NULL,
93
};
94

95
static struct attribute *node_targ_access_node_attrs[] = {
96
	NULL,
97
};
98

99
static const struct attribute_group initiators = {
100
	.name	= "initiators",
101
	.attrs	= node_init_access_node_attrs,
102
};
103

104
static const struct attribute_group targets = {
105
	.name	= "targets",
106
	.attrs	= node_targ_access_node_attrs,
107
};
108

109
static const struct attribute_group *node_access_node_groups[] = {
110
	&initiators,
111
	&targets,
112
	NULL,
113
};
114

115
#ifdef CONFIG_MEMORY_HOTPLUG
116
static BLOCKING_NOTIFIER_HEAD(node_chain);
117

118
int register_node_notifier(struct notifier_block *nb)
119
{
120
	return blocking_notifier_chain_register(&node_chain, nb);
121
}
122
EXPORT_SYMBOL(register_node_notifier);
123

124
void unregister_node_notifier(struct notifier_block *nb)
125
{
126
	blocking_notifier_chain_unregister(&node_chain, nb);
127
}
128
EXPORT_SYMBOL(unregister_node_notifier);
129

130
int node_notify(unsigned long val, void *v)
131
{
132
	return blocking_notifier_call_chain(&node_chain, val, v);
133
}
134
#endif
135

136
static void node_remove_accesses(struct node *node)
137
{
138
	struct node_access_nodes *c, *cnext;
139

140
	list_for_each_entry_safe(c, cnext, &node->access_list, list_node) {
141
		list_del(&c->list_node);
142
		device_unregister(&c->dev);
143
	}
144
}
145

146
static void node_access_release(struct device *dev)
147
{
148
	kfree(to_access_nodes(dev));
149
}
150

151
static struct node_access_nodes *node_init_node_access(struct node *node,
152
						       enum access_coordinate_class access)
153
{
154
	struct node_access_nodes *access_node;
155
	struct device *dev;
156

157
	list_for_each_entry(access_node, &node->access_list, list_node)
158
		if (access_node->access == access)
159
			return access_node;
160

161
	access_node = kzalloc(sizeof(*access_node), GFP_KERNEL);
162
	if (!access_node)
163
		return NULL;
164

165
	access_node->access = access;
166
	dev = &access_node->dev;
167
	dev->parent = &node->dev;
168
	dev->release = node_access_release;
169
	dev->groups = node_access_node_groups;
170
	if (dev_set_name(dev, "access%u", access))
171
		goto free;
172

173
	if (device_register(dev))
174
		goto free_name;
175

176
	pm_runtime_no_callbacks(dev);
177
	list_add_tail(&access_node->list_node, &node->access_list);
178
	return access_node;
179
free_name:
180
	kfree_const(dev->kobj.name);
181
free:
182
	kfree(access_node);
183
	return NULL;
184
}
185

186
#ifdef CONFIG_HMEM_REPORTING
187
#define ACCESS_ATTR(property)						\
188
static ssize_t property##_show(struct device *dev,			\
189
			   struct device_attribute *attr,		\
190
			   char *buf)					\
191
{									\
192
	return sysfs_emit(buf, "%u\n",					\
193
			  to_access_nodes(dev)->coord.property);	\
194
}									\
195
static DEVICE_ATTR_RO(property)
196

197
ACCESS_ATTR(read_bandwidth);
198
ACCESS_ATTR(read_latency);
199
ACCESS_ATTR(write_bandwidth);
200
ACCESS_ATTR(write_latency);
201

202
static struct attribute *access_attrs[] = {
203
	&dev_attr_read_bandwidth.attr,
204
	&dev_attr_read_latency.attr,
205
	&dev_attr_write_bandwidth.attr,
206
	&dev_attr_write_latency.attr,
207
	NULL,
208
};
209

210
/**
211
 * node_set_perf_attrs - Set the performance values for given access class
212
 * @nid: Node identifier to be set
213
 * @coord: Heterogeneous memory performance coordinates
214
 * @access: The access class the for the given attributes
215
 */
216
void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord,
217
			 enum access_coordinate_class access)
218
{
219
	struct node_access_nodes *c;
220
	struct node *node;
221
	int i;
222

223
	if (WARN_ON_ONCE(!node_online(nid)))
224
		return;
225

226
	node = node_devices[nid];
227
	c = node_init_node_access(node, access);
228
	if (!c)
229
		return;
230

231
	c->coord = *coord;
232
	for (i = 0; access_attrs[i] != NULL; i++) {
233
		if (sysfs_add_file_to_group(&c->dev.kobj, access_attrs[i],
234
					    "initiators")) {
235
			pr_info("failed to add performance attribute to node %d\n",
236
				nid);
237
			break;
238
		}
239
	}
240

241
	/* When setting CPU access coordinates, update mempolicy */
242
	if (access == ACCESS_COORDINATE_CPU) {
243
		if (mempolicy_set_node_perf(nid, coord)) {
244
			pr_info("failed to set mempolicy attrs for node %d\n",
245
				nid);
246
		}
247
	}
248
}
249
EXPORT_SYMBOL_GPL(node_set_perf_attrs);
250

251
/**
252
 * node_update_perf_attrs - Update the performance values for given access class
253
 * @nid: Node identifier to be updated
254
 * @coord: Heterogeneous memory performance coordinates
255
 * @access: The access class for the given attributes
256
 */
257
void node_update_perf_attrs(unsigned int nid, struct access_coordinate *coord,
258
			    enum access_coordinate_class access)
259
{
260
	struct node_access_nodes *access_node;
261
	struct node *node;
262
	int i;
263

264
	if (WARN_ON_ONCE(!node_online(nid)))
265
		return;
266

267
	node = node_devices[nid];
268
	list_for_each_entry(access_node, &node->access_list, list_node) {
269
		if (access_node->access != access)
270
			continue;
271

272
		access_node->coord = *coord;
273
		for (i = 0; access_attrs[i]; i++) {
274
			sysfs_notify(&access_node->dev.kobj,
275
				     NULL, access_attrs[i]->name);
276
		}
277
		break;
278
	}
279

280
	/* When setting CPU access coordinates, update mempolicy */
281
	if (access != ACCESS_COORDINATE_CPU)
282
		return;
283

284
	if (mempolicy_set_node_perf(nid, coord))
285
		pr_info("failed to set mempolicy attrs for node %d\n", nid);
286
}
287
EXPORT_SYMBOL_GPL(node_update_perf_attrs);
288

289
/**
290
 * struct node_cache_info - Internal tracking for memory node caches
291
 * @dev:	Device represeting the cache level
292
 * @node:	List element for tracking in the node
293
 * @cache_attrs:Attributes for this cache level
294
 */
295
struct node_cache_info {
296
	struct device dev;
297
	struct list_head node;
298
	struct node_cache_attrs cache_attrs;
299
};
300
#define to_cache_info(device) container_of(device, struct node_cache_info, dev)
301

302
#define CACHE_ATTR(name, fmt) 						\
303
static ssize_t name##_show(struct device *dev,				\
304
			   struct device_attribute *attr,		\
305
			   char *buf)					\
306
{									\
307
	return sysfs_emit(buf, fmt "\n",				\
308
			  to_cache_info(dev)->cache_attrs.name);	\
309
}									\
310
static DEVICE_ATTR_RO(name);
311

312
CACHE_ATTR(size, "%llu")
313
CACHE_ATTR(line_size, "%u")
314
CACHE_ATTR(indexing, "%u")
315
CACHE_ATTR(write_policy, "%u")
316
CACHE_ATTR(address_mode, "%#x")
317

318
static struct attribute *cache_attrs[] = {
319
	&dev_attr_indexing.attr,
320
	&dev_attr_size.attr,
321
	&dev_attr_line_size.attr,
322
	&dev_attr_write_policy.attr,
323
	&dev_attr_address_mode.attr,
324
	NULL,
325
};
326
ATTRIBUTE_GROUPS(cache);
327

328
static void node_cache_release(struct device *dev)
329
{
330
	kfree(dev);
331
}
332

333
static void node_cacheinfo_release(struct device *dev)
334
{
335
	struct node_cache_info *info = to_cache_info(dev);
336
	kfree(info);
337
}
338

339
static void node_init_cache_dev(struct node *node)
340
{
341
	struct device *dev;
342

343
	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
344
	if (!dev)
345
		return;
346

347
	device_initialize(dev);
348
	dev->parent = &node->dev;
349
	dev->release = node_cache_release;
350
	if (dev_set_name(dev, "memory_side_cache"))
351
		goto put_device;
352

353
	if (device_add(dev))
354
		goto put_device;
355

356
	pm_runtime_no_callbacks(dev);
357
	node->cache_dev = dev;
358
	return;
359
put_device:
360
	put_device(dev);
361
}
362

363
/**
364
 * node_add_cache() - add cache attribute to a memory node
365
 * @nid: Node identifier that has new cache attributes
366
 * @cache_attrs: Attributes for the cache being added
367
 */
368
void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs)
369
{
370
	struct node_cache_info *info;
371
	struct device *dev;
372
	struct node *node;
373

374
	if (!node_online(nid) || !node_devices[nid])
375
		return;
376

377
	node = node_devices[nid];
378
	list_for_each_entry(info, &node->cache_attrs, node) {
379
		if (info->cache_attrs.level == cache_attrs->level) {
380
			dev_warn(&node->dev,
381
				"attempt to add duplicate cache level:%d\n",
382
				cache_attrs->level);
383
			return;
384
		}
385
	}
386

387
	if (!node->cache_dev)
388
		node_init_cache_dev(node);
389
	if (!node->cache_dev)
390
		return;
391

392
	info = kzalloc(sizeof(*info), GFP_KERNEL);
393
	if (!info)
394
		return;
395

396
	dev = &info->dev;
397
	device_initialize(dev);
398
	dev->parent = node->cache_dev;
399
	dev->release = node_cacheinfo_release;
400
	dev->groups = cache_groups;
401
	if (dev_set_name(dev, "index%d", cache_attrs->level))
402
		goto put_device;
403

404
	info->cache_attrs = *cache_attrs;
405
	if (device_add(dev)) {
406
		dev_warn(&node->dev, "failed to add cache level:%d\n",
407
			 cache_attrs->level);
408
		goto put_device;
409
	}
410
	pm_runtime_no_callbacks(dev);
411
	list_add_tail(&info->node, &node->cache_attrs);
412
	return;
413
put_device:
414
	put_device(dev);
415
}
416

417
static void node_remove_caches(struct node *node)
418
{
419
	struct node_cache_info *info, *next;
420

421
	if (!node->cache_dev)
422
		return;
423

424
	list_for_each_entry_safe(info, next, &node->cache_attrs, node) {
425
		list_del(&info->node);
426
		device_unregister(&info->dev);
427
	}
428
	device_unregister(node->cache_dev);
429
}
430

431
static void node_init_caches(unsigned int nid)
432
{
433
	INIT_LIST_HEAD(&node_devices[nid]->cache_attrs);
434
}
435
#else
436
static void node_init_caches(unsigned int nid) { }
437
static void node_remove_caches(struct node *node) { }
438
#endif
439

440
#define K(x) ((x) << (PAGE_SHIFT - 10))
441
static ssize_t node_read_meminfo(struct device *dev,
442
			struct device_attribute *attr, char *buf)
443
{
444
	int len = 0;
445
	int nid = dev->id;
446
	struct pglist_data *pgdat = NODE_DATA(nid);
447
	struct sysinfo i;
448
	unsigned long sreclaimable, sunreclaimable;
449
	unsigned long swapcached = 0;
450

451
	si_meminfo_node(&i, nid);
452
	sreclaimable = node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B);
453
	sunreclaimable = node_page_state_pages(pgdat, NR_SLAB_UNRECLAIMABLE_B);
454
#ifdef CONFIG_SWAP
455
	swapcached = node_page_state_pages(pgdat, NR_SWAPCACHE);
456
#endif
457
	len = sysfs_emit_at(buf, len,
458
			    "Node %d MemTotal:       %8lu kB\n"
459
			    "Node %d MemFree:        %8lu kB\n"
460
			    "Node %d MemUsed:        %8lu kB\n"
461
			    "Node %d SwapCached:     %8lu kB\n"
462
			    "Node %d Active:         %8lu kB\n"
463
			    "Node %d Inactive:       %8lu kB\n"
464
			    "Node %d Active(anon):   %8lu kB\n"
465
			    "Node %d Inactive(anon): %8lu kB\n"
466
			    "Node %d Active(file):   %8lu kB\n"
467
			    "Node %d Inactive(file): %8lu kB\n"
468
			    "Node %d Unevictable:    %8lu kB\n"
469
			    "Node %d Mlocked:        %8lu kB\n",
470
			    nid, K(i.totalram),
471
			    nid, K(i.freeram),
472
			    nid, K(i.totalram - i.freeram),
473
			    nid, K(swapcached),
474
			    nid, K(node_page_state(pgdat, NR_ACTIVE_ANON) +
475
				   node_page_state(pgdat, NR_ACTIVE_FILE)),
476
			    nid, K(node_page_state(pgdat, NR_INACTIVE_ANON) +
477
				   node_page_state(pgdat, NR_INACTIVE_FILE)),
478
			    nid, K(node_page_state(pgdat, NR_ACTIVE_ANON)),
479
			    nid, K(node_page_state(pgdat, NR_INACTIVE_ANON)),
480
			    nid, K(node_page_state(pgdat, NR_ACTIVE_FILE)),
481
			    nid, K(node_page_state(pgdat, NR_INACTIVE_FILE)),
482
			    nid, K(node_page_state(pgdat, NR_UNEVICTABLE)),
483
			    nid, K(sum_zone_node_page_state(nid, NR_MLOCK)));
484

485
#ifdef CONFIG_HIGHMEM
486
	len += sysfs_emit_at(buf, len,
487
			     "Node %d HighTotal:      %8lu kB\n"
488
			     "Node %d HighFree:       %8lu kB\n"
489
			     "Node %d LowTotal:       %8lu kB\n"
490
			     "Node %d LowFree:        %8lu kB\n",
491
			     nid, K(i.totalhigh),
492
			     nid, K(i.freehigh),
493
			     nid, K(i.totalram - i.totalhigh),
494
			     nid, K(i.freeram - i.freehigh));
495
#endif
496
	len += sysfs_emit_at(buf, len,
497
			     "Node %d Dirty:          %8lu kB\n"
498
			     "Node %d Writeback:      %8lu kB\n"
499
			     "Node %d FilePages:      %8lu kB\n"
500
			     "Node %d Mapped:         %8lu kB\n"
501
			     "Node %d AnonPages:      %8lu kB\n"
502
			     "Node %d Shmem:          %8lu kB\n"
503
			     "Node %d KernelStack:    %8lu kB\n"
504
#ifdef CONFIG_SHADOW_CALL_STACK
505
			     "Node %d ShadowCallStack:%8lu kB\n"
506
#endif
507
			     "Node %d PageTables:     %8lu kB\n"
508
			     "Node %d SecPageTables:  %8lu kB\n"
509
			     "Node %d NFS_Unstable:   %8lu kB\n"
510
			     "Node %d Bounce:         %8lu kB\n"
511
			     "Node %d WritebackTmp:   %8lu kB\n"
512
			     "Node %d KReclaimable:   %8lu kB\n"
513
			     "Node %d Slab:           %8lu kB\n"
514
			     "Node %d SReclaimable:   %8lu kB\n"
515
			     "Node %d SUnreclaim:     %8lu kB\n"
516
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
517
			     "Node %d AnonHugePages:  %8lu kB\n"
518
			     "Node %d ShmemHugePages: %8lu kB\n"
519
			     "Node %d ShmemPmdMapped: %8lu kB\n"
520
			     "Node %d FileHugePages:  %8lu kB\n"
521
			     "Node %d FilePmdMapped:  %8lu kB\n"
522
#endif
523
#ifdef CONFIG_UNACCEPTED_MEMORY
524
			     "Node %d Unaccepted:     %8lu kB\n"
525
#endif
526
			     ,
527
			     nid, K(node_page_state(pgdat, NR_FILE_DIRTY)),
528
			     nid, K(node_page_state(pgdat, NR_WRITEBACK)),
529
			     nid, K(node_page_state(pgdat, NR_FILE_PAGES)),
530
			     nid, K(node_page_state(pgdat, NR_FILE_MAPPED)),
531
			     nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
532
			     nid, K(i.sharedram),
533
			     nid, node_page_state(pgdat, NR_KERNEL_STACK_KB),
534
#ifdef CONFIG_SHADOW_CALL_STACK
535
			     nid, node_page_state(pgdat, NR_KERNEL_SCS_KB),
536
#endif
537
			     nid, K(node_page_state(pgdat, NR_PAGETABLE)),
538
			     nid, K(node_page_state(pgdat, NR_SECONDARY_PAGETABLE)),
539
			     nid, 0UL,
540
			     nid, 0UL,
541
			     nid, 0UL,
542
			     nid, K(sreclaimable +
543
				    node_page_state(pgdat, NR_KERNEL_MISC_RECLAIMABLE)),
544
			     nid, K(sreclaimable + sunreclaimable),
545
			     nid, K(sreclaimable),
546
			     nid, K(sunreclaimable)
547
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
548
			     ,
549
			     nid, K(node_page_state(pgdat, NR_ANON_THPS)),
550
			     nid, K(node_page_state(pgdat, NR_SHMEM_THPS)),
551
			     nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)),
552
			     nid, K(node_page_state(pgdat, NR_FILE_THPS)),
553
			     nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED))
554
#endif
555
#ifdef CONFIG_UNACCEPTED_MEMORY
556
			     ,
557
			     nid, K(sum_zone_node_page_state(nid, NR_UNACCEPTED))
558
#endif
559
			    );
560
	len += hugetlb_report_node_meminfo(buf, len, nid);
561
	return len;
562
}
563

564
#undef K
565
static DEVICE_ATTR(meminfo, 0444, node_read_meminfo, NULL);
566

567
static ssize_t node_read_numastat(struct device *dev,
568
				  struct device_attribute *attr, char *buf)
569
{
570
	fold_vm_numa_events();
571
	return sysfs_emit(buf,
572
			  "numa_hit %lu\n"
573
			  "numa_miss %lu\n"
574
			  "numa_foreign %lu\n"
575
			  "interleave_hit %lu\n"
576
			  "local_node %lu\n"
577
			  "other_node %lu\n",
578
			  sum_zone_numa_event_state(dev->id, NUMA_HIT),
579
			  sum_zone_numa_event_state(dev->id, NUMA_MISS),
580
			  sum_zone_numa_event_state(dev->id, NUMA_FOREIGN),
581
			  sum_zone_numa_event_state(dev->id, NUMA_INTERLEAVE_HIT),
582
			  sum_zone_numa_event_state(dev->id, NUMA_LOCAL),
583
			  sum_zone_numa_event_state(dev->id, NUMA_OTHER));
584
}
585
static DEVICE_ATTR(numastat, 0444, node_read_numastat, NULL);
586

587
static ssize_t node_read_vmstat(struct device *dev,
588
				struct device_attribute *attr, char *buf)
589
{
590
	int nid = dev->id;
591
	struct pglist_data *pgdat = NODE_DATA(nid);
592
	int i;
593
	int len = 0;
594

595
	for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
596
		len += sysfs_emit_at(buf, len, "%s %lu\n",
597
				     zone_stat_name(i),
598
				     sum_zone_node_page_state(nid, i));
599

600
#ifdef CONFIG_NUMA
601
	fold_vm_numa_events();
602
	for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++)
603
		len += sysfs_emit_at(buf, len, "%s %lu\n",
604
				     numa_stat_name(i),
605
				     sum_zone_numa_event_state(nid, i));
606

607
#endif
608
	for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
609
		unsigned long pages = node_page_state_pages(pgdat, i);
610

611
		if (vmstat_item_print_in_thp(i))
612
			pages /= HPAGE_PMD_NR;
613
		len += sysfs_emit_at(buf, len, "%s %lu\n", node_stat_name(i),
614
				     pages);
615
	}
616

617
	return len;
618
}
619
static DEVICE_ATTR(vmstat, 0444, node_read_vmstat, NULL);
620

621
static ssize_t node_read_distance(struct device *dev,
622
				  struct device_attribute *attr, char *buf)
623
{
624
	int nid = dev->id;
625
	int len = 0;
626
	int i;
627

628
	/*
629
	 * buf is currently PAGE_SIZE in length and each node needs 4 chars
630
	 * at the most (distance + space or newline).
631
	 */
632
	BUILD_BUG_ON(MAX_NUMNODES * 4 > PAGE_SIZE);
633

634
	for_each_online_node(i) {
635
		len += sysfs_emit_at(buf, len, "%s%d",
636
				     i ? " " : "", node_distance(nid, i));
637
	}
638

639
	len += sysfs_emit_at(buf, len, "\n");
640
	return len;
641
}
642
static DEVICE_ATTR(distance, 0444, node_read_distance, NULL);
643

644
static struct attribute *node_dev_attrs[] = {
645
	&dev_attr_meminfo.attr,
646
	&dev_attr_numastat.attr,
647
	&dev_attr_distance.attr,
648
	&dev_attr_vmstat.attr,
649
	NULL
650
};
651

652
static const struct bin_attribute *node_dev_bin_attrs[] = {
653
	&bin_attr_cpumap,
654
	&bin_attr_cpulist,
655
	NULL
656
};
657

658
static const struct attribute_group node_dev_group = {
659
	.attrs = node_dev_attrs,
660
	.bin_attrs = node_dev_bin_attrs,
661
};
662

663
static const struct attribute_group *node_dev_groups[] = {
664
	&node_dev_group,
665
#ifdef CONFIG_HAVE_ARCH_NODE_DEV_GROUP
666
	&arch_node_dev_group,
667
#endif
668
#ifdef CONFIG_MEMORY_FAILURE
669
	&memory_failure_attr_group,
670
#endif
671
	NULL
672
};
673

674
static void node_device_release(struct device *dev)
675
{
676
	kfree(to_node(dev));
677
}
678

679
/*
680
 * register_node - Setup a sysfs device for a node.
681
 * @num - Node number to use when creating the device.
682
 *
683
 * Initialize and register the node device.
684
 */
685
static int register_node(struct node *node, int num)
686
{
687
	int error;
688

689
	node->dev.id = num;
690
	node->dev.bus = &node_subsys;
691
	node->dev.release = node_device_release;
692
	node->dev.groups = node_dev_groups;
693
	error = device_register(&node->dev);
694

695
	if (error) {
696
		put_device(&node->dev);
697
	} else {
698
		hugetlb_register_node(node);
699
		compaction_register_node(node);
700
		reclaim_register_node(node);
701
	}
702

703
	return error;
704
}
705

706
/**
707
 * unregister_node - unregister a node device
708
 * @node: node going away
709
 *
710
 * Unregisters a node device @node.  All the devices on the node must be
711
 * unregistered before calling this function.
712
 */
713
void unregister_node(struct node *node)
714
{
715
	hugetlb_unregister_node(node);
716
	compaction_unregister_node(node);
717
	reclaim_unregister_node(node);
718
	node_remove_accesses(node);
719
	node_remove_caches(node);
720
	device_unregister(&node->dev);
721
}
722

723
struct node *node_devices[MAX_NUMNODES];
724

725
/*
726
 * register cpu under node
727
 */
728
int register_cpu_under_node(unsigned int cpu, unsigned int nid)
729
{
730
	int ret;
731
	struct device *obj;
732

733
	if (!node_online(nid))
734
		return 0;
735

736
	obj = get_cpu_device(cpu);
737
	if (!obj)
738
		return 0;
739

740
	ret = sysfs_create_link(&node_devices[nid]->dev.kobj,
741
				&obj->kobj,
742
				kobject_name(&obj->kobj));
743
	if (ret)
744
		return ret;
745

746
	return sysfs_create_link(&obj->kobj,
747
				 &node_devices[nid]->dev.kobj,
748
				 kobject_name(&node_devices[nid]->dev.kobj));
749
}
750

751
/**
752
 * register_memory_node_under_compute_node - link memory node to its compute
753
 *					     node for a given access class.
754
 * @mem_nid:	Memory node number
755
 * @cpu_nid:	Cpu  node number
756
 * @access:	Access class to register
757
 *
758
 * Description:
759
 * 	For use with platforms that may have separate memory and compute nodes.
760
 * 	This function will export node relationships linking which memory
761
 * 	initiator nodes can access memory targets at a given ranked access
762
 * 	class.
763
 */
764
int register_memory_node_under_compute_node(unsigned int mem_nid,
765
					    unsigned int cpu_nid,
766
					    enum access_coordinate_class access)
767
{
768
	struct node *init_node, *targ_node;
769
	struct node_access_nodes *initiator, *target;
770
	int ret;
771

772
	if (!node_online(cpu_nid) || !node_online(mem_nid))
773
		return -ENODEV;
774

775
	init_node = node_devices[cpu_nid];
776
	targ_node = node_devices[mem_nid];
777
	initiator = node_init_node_access(init_node, access);
778
	target = node_init_node_access(targ_node, access);
779
	if (!initiator || !target)
780
		return -ENOMEM;
781

782
	ret = sysfs_add_link_to_group(&initiator->dev.kobj, "targets",
783
				      &targ_node->dev.kobj,
784
				      dev_name(&targ_node->dev));
785
	if (ret)
786
		return ret;
787

788
	ret = sysfs_add_link_to_group(&target->dev.kobj, "initiators",
789
				      &init_node->dev.kobj,
790
				      dev_name(&init_node->dev));
791
	if (ret)
792
		goto err;
793

794
	return 0;
795
 err:
796
	sysfs_remove_link_from_group(&initiator->dev.kobj, "targets",
797
				     dev_name(&targ_node->dev));
798
	return ret;
799
}
800

801
int unregister_cpu_under_node(unsigned int cpu, unsigned int nid)
802
{
803
	struct device *obj;
804

805
	if (!node_online(nid))
806
		return 0;
807

808
	obj = get_cpu_device(cpu);
809
	if (!obj)
810
		return 0;
811

812
	sysfs_remove_link(&node_devices[nid]->dev.kobj,
813
			  kobject_name(&obj->kobj));
814
	sysfs_remove_link(&obj->kobj,
815
			  kobject_name(&node_devices[nid]->dev.kobj));
816

817
	return 0;
818
}
819

820
#ifdef CONFIG_MEMORY_HOTPLUG
821
static void do_register_memory_block_under_node(int nid,
822
						struct memory_block *mem_blk)
823
{
824
	int ret;
825

826
	ret = sysfs_create_link_nowarn(&node_devices[nid]->dev.kobj,
827
				       &mem_blk->dev.kobj,
828
				       kobject_name(&mem_blk->dev.kobj));
829
	if (ret && ret != -EEXIST)
830
		dev_err_ratelimited(&node_devices[nid]->dev,
831
				    "can't create link to %s in sysfs (%d)\n",
832
				    kobject_name(&mem_blk->dev.kobj), ret);
833

834
	ret = sysfs_create_link_nowarn(&mem_blk->dev.kobj,
835
				&node_devices[nid]->dev.kobj,
836
				kobject_name(&node_devices[nid]->dev.kobj));
837
	if (ret && ret != -EEXIST)
838
		dev_err_ratelimited(&mem_blk->dev,
839
				    "can't create link to %s in sysfs (%d)\n",
840
				    kobject_name(&node_devices[nid]->dev.kobj),
841
				    ret);
842
}
843

844
/*
845
 * During hotplug we know that all pages in the memory block belong to the same
846
 * node.
847
 */
848
static int register_mem_block_under_node_hotplug(struct memory_block *mem_blk,
849
						 void *arg)
850
{
851
	int nid = *(int *)arg;
852

853
	do_register_memory_block_under_node(nid, mem_blk);
854
	return 0;
855
}
856

857
/*
858
 * Unregister a memory block device under the node it spans. Memory blocks
859
 * with multiple nodes cannot be offlined and therefore also never be removed.
860
 */
861
void unregister_memory_block_under_nodes(struct memory_block *mem_blk)
862
{
863
	if (mem_blk->nid == NUMA_NO_NODE)
864
		return;
865

866
	sysfs_remove_link(&node_devices[mem_blk->nid]->dev.kobj,
867
			  kobject_name(&mem_blk->dev.kobj));
868
	sysfs_remove_link(&mem_blk->dev.kobj,
869
			  kobject_name(&node_devices[mem_blk->nid]->dev.kobj));
870
}
871

872
/* register all memory blocks under the corresponding nodes */
873
static void register_memory_blocks_under_nodes(void)
874
{
875
	struct memblock_region *r;
876

877
	for_each_mem_region(r) {
878
		const unsigned long start_block_id = phys_to_block_id(r->base);
879
		const unsigned long end_block_id = phys_to_block_id(r->base + r->size - 1);
880
		const int nid = memblock_get_region_node(r);
881
		unsigned long block_id;
882

883
		if (!node_online(nid))
884
			continue;
885

886
		for (block_id = start_block_id; block_id <= end_block_id; block_id++) {
887
			struct memory_block *mem;
888

889
			mem = find_memory_block_by_id(block_id);
890
			if (!mem)
891
				continue;
892

893
			memory_block_add_nid_early(mem, nid);
894
			do_register_memory_block_under_node(nid, mem);
895
			put_device(&mem->dev);
896
		}
897

898
	}
899
}
900

901
void register_memory_blocks_under_node_hotplug(int nid, unsigned long start_pfn,
902
					       unsigned long end_pfn)
903
{
904
	walk_memory_blocks(PFN_PHYS(start_pfn), PFN_PHYS(end_pfn - start_pfn),
905
			   (void *)&nid, register_mem_block_under_node_hotplug);
906
	return;
907
}
908
#endif /* CONFIG_MEMORY_HOTPLUG */
909

910
int register_one_node(int nid)
911
{
912
	int error;
913
	int cpu;
914
	struct node *node;
915

916
	node = kzalloc(sizeof(struct node), GFP_KERNEL);
917
	if (!node)
918
		return -ENOMEM;
919

920
	INIT_LIST_HEAD(&node->access_list);
921
	node_devices[nid] = node;
922

923
	error = register_node(node_devices[nid], nid);
924
	if (error) {
925
		node_devices[nid] = NULL;
926
		return error;
927
	}
928

929
	/* link cpu under this node */
930
	for_each_present_cpu(cpu) {
931
		if (cpu_to_node(cpu) == nid)
932
			register_cpu_under_node(cpu, nid);
933
	}
934

935
	node_init_caches(nid);
936

937
	return error;
938
}
939

940
void unregister_one_node(int nid)
941
{
942
	if (!node_devices[nid])
943
		return;
944

945
	unregister_node(node_devices[nid]);
946
	node_devices[nid] = NULL;
947
}
948

949
/*
950
 * node states attributes
951
 */
952

953
struct node_attr {
954
	struct device_attribute attr;
955
	enum node_states state;
956
};
957

958
static ssize_t show_node_state(struct device *dev,
959
			       struct device_attribute *attr, char *buf)
960
{
961
	struct node_attr *na = container_of(attr, struct node_attr, attr);
962

963
	return sysfs_emit(buf, "%*pbl\n",
964
			  nodemask_pr_args(&node_states[na->state]));
965
}
966

967
#define _NODE_ATTR(name, state) \
968
	{ __ATTR(name, 0444, show_node_state, NULL), state }
969

970
static struct node_attr node_state_attr[] = {
971
	[N_POSSIBLE] = _NODE_ATTR(possible, N_POSSIBLE),
972
	[N_ONLINE] = _NODE_ATTR(online, N_ONLINE),
973
	[N_NORMAL_MEMORY] = _NODE_ATTR(has_normal_memory, N_NORMAL_MEMORY),
974
#ifdef CONFIG_HIGHMEM
975
	[N_HIGH_MEMORY] = _NODE_ATTR(has_high_memory, N_HIGH_MEMORY),
976
#endif
977
	[N_MEMORY] = _NODE_ATTR(has_memory, N_MEMORY),
978
	[N_CPU] = _NODE_ATTR(has_cpu, N_CPU),
979
	[N_GENERIC_INITIATOR] = _NODE_ATTR(has_generic_initiator,
980
					   N_GENERIC_INITIATOR),
981
};
982

983
static struct attribute *node_state_attrs[] = {
984
	&node_state_attr[N_POSSIBLE].attr.attr,
985
	&node_state_attr[N_ONLINE].attr.attr,
986
	&node_state_attr[N_NORMAL_MEMORY].attr.attr,
987
#ifdef CONFIG_HIGHMEM
988
	&node_state_attr[N_HIGH_MEMORY].attr.attr,
989
#endif
990
	&node_state_attr[N_MEMORY].attr.attr,
991
	&node_state_attr[N_CPU].attr.attr,
992
	&node_state_attr[N_GENERIC_INITIATOR].attr.attr,
993
	NULL
994
};
995

996
static const struct attribute_group memory_root_attr_group = {
997
	.attrs = node_state_attrs,
998
};
999

1000
static const struct attribute_group *cpu_root_attr_groups[] = {
1001
	&memory_root_attr_group,
1002
	NULL,
1003
};
1004

1005
void __init node_dev_init(void)
1006
{
1007
	int ret, i;
1008

1009
 	BUILD_BUG_ON(ARRAY_SIZE(node_state_attr) != NR_NODE_STATES);
1010
 	BUILD_BUG_ON(ARRAY_SIZE(node_state_attrs)-1 != NR_NODE_STATES);
1011

1012
	ret = subsys_system_register(&node_subsys, cpu_root_attr_groups);
1013
	if (ret)
1014
		panic("%s() failed to register subsystem: %d\n", __func__, ret);
1015

1016
	/*
1017
	 * Create all node devices, which will properly link the node
1018
	 * to already created cpu devices.
1019
	 */
1020
	for_each_online_node(i) {
1021
		ret =  register_one_node(i);
1022
		if (ret)
1023
			panic("%s() failed to add node: %d\n", __func__, ret);
1024
	}
1025

1026
	register_memory_blocks_under_nodes();
1027
}
1028

1029