2 * Copyright (c) 2015 - 2016, NVIDIA CORPORATION. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 #include <linux/types.h>
15 #include <linux/mutex.h>
16 #include <linux/err.h>
17 #include <linux/module.h>
18 #include <linux/platform/tegra/emc_bwmgr.h>
19 #include <linux/platform/tegra/isomgr.h>
20 #include <linux/debugfs.h>
22 u8 bwmgr_dram_efficiency;
23 u32 *bwmgr_dram_iso_eff_table;
24 int bwmgr_iso_bw_percentage;
25 enum bwmgr_dram_types bwmgr_dram_type;
26 int emc_to_dram_freq_factor;
28 #define IS_HANDLE_VALID(x) ((x >= bwmgr.bwmgr_client) && \
29 (x < bwmgr.bwmgr_client + TEGRA_BWMGR_CLIENT_COUNT))
31 struct tegra_bwmgr_client {
35 unsigned long iso_cap;
40 /* TODO: Manage client state in a dynamic list */
42 struct tegra_bwmgr_client bwmgr_client[TEGRA_BWMGR_CLIENT_COUNT];
44 unsigned long emc_min_rate;
45 unsigned long emc_max_rate;
47 struct task_struct *task;
51 static bool clk_update_disabled;
56 unsigned long non_iso_cap;
57 unsigned long iso_cap;
59 unsigned long total_bw_aftr_eff;
60 unsigned long iso_bw_aftr_eff;
61 unsigned long calc_freq;
62 unsigned long req_freq;
65 static void bwmgr_debugfs_init(void);
67 static inline void bwmgr_lock(void)
69 BUG_ON(bwmgr.task == current); /* disallow rentrance, avoid deadlock */
70 mutex_lock(&bwmgr.lock);
74 static inline void bwmgr_unlock(void)
76 BUG_ON(bwmgr.task != current); /* detect mismatched calls */
78 mutex_unlock(&bwmgr.lock);
81 /* call with bwmgr lock held except during init*/
82 static void purge_client(struct tegra_bwmgr_client *handle)
86 handle->cap = bwmgr.emc_max_rate;
87 handle->iso_cap = bwmgr.emc_max_rate;
92 static unsigned long bwmgr_apply_efficiency(
93 unsigned long total_bw, unsigned long iso_bw,
94 unsigned long max_rate, u64 usage_flags,
95 unsigned long *iso_bw_min);
97 /* call with bwmgr lock held */
98 static int bwmgr_update_clk(void)
101 unsigned long bw = 0;
102 unsigned long iso_bw = 0;
103 unsigned long non_iso_cap = bwmgr.emc_max_rate;
104 unsigned long iso_cap = bwmgr.emc_max_rate;
105 unsigned long floor = 0;
106 unsigned long iso_bw_min;
107 u64 iso_client_flags = 0;
110 /* sizeof(iso_client_flags) */
111 BUILD_BUG_ON(TEGRA_BWMGR_CLIENT_COUNT > 64);
112 BUG_ON(bwmgr.task != current); /* check that lock is held */
114 for (i = 0; i < TEGRA_BWMGR_CLIENT_COUNT; i++) {
115 bw += bwmgr.bwmgr_client[i].bw;
116 bw = min(bw, bwmgr.emc_max_rate);
118 if (bwmgr.bwmgr_client[i].iso_bw > 0) {
119 iso_client_flags |= BIT(i);
120 iso_bw += bwmgr.bwmgr_client[i].iso_bw;
121 iso_bw = min(iso_bw, bwmgr.emc_max_rate);
124 non_iso_cap = min(non_iso_cap, bwmgr.bwmgr_client[i].cap);
125 iso_cap = min(iso_cap, bwmgr.bwmgr_client[i].iso_cap);
126 floor = max(floor, bwmgr.bwmgr_client[i].floor);
129 debug_info.iso_bw = iso_bw;
130 debug_info.floor = floor;
131 debug_info.iso_cap = iso_cap;
132 debug_info.non_iso_cap = non_iso_cap;
134 bw = bwmgr_apply_efficiency(
135 bw, iso_bw, bwmgr.emc_max_rate,
136 iso_client_flags, &iso_bw_min);
137 debug_info.total_bw_aftr_eff = bw;
138 debug_info.iso_bw_aftr_eff = iso_bw_min;
139 iso_bw_min = clk_round_rate(bwmgr.emc_clk, iso_bw_min);
140 floor = min(floor, bwmgr.emc_max_rate);
142 bw = min(bw, min(iso_cap, max(non_iso_cap, iso_bw_min)));
143 debug_info.calc_freq = bw;
144 bw = clk_round_rate(bwmgr.emc_clk, bw);
145 debug_info.req_freq = bw;
146 if (bw == tegra_bwmgr_get_emc_rate())
149 ret = clk_set_rate(bwmgr.emc_clk, bw);
152 ("bwmgr: clk_set_rate failed for freq %lu Hz with errno %d\n",
158 struct tegra_bwmgr_client *tegra_bwmgr_register(
159 enum tegra_bwmgr_client_id client)
161 if ((client >= TEGRA_BWMGR_CLIENT_COUNT) || (client < 0)) {
162 pr_err("bwmgr: invalid client id %d tried to register",
165 return ERR_PTR(-EINVAL);
169 (bwmgr.bwmgr_client + client)->refcount++;
171 return (bwmgr.bwmgr_client + client);
173 EXPORT_SYMBOL_GPL(tegra_bwmgr_register);
175 void tegra_bwmgr_unregister(struct tegra_bwmgr_client *handle)
177 if (!IS_HANDLE_VALID(handle)) {
185 if (handle->refcount <= 0) {
186 if (handle->refcount < 0) {
187 pr_err("bwmgr: Mismatched unregister call, client %ld\n",
188 handle - bwmgr.bwmgr_client);
191 purge_client(handle);
196 EXPORT_SYMBOL_GPL(tegra_bwmgr_unregister);
198 unsigned long tegra_bwmgr_get_max_emc_rate(void)
200 return bwmgr.emc_max_rate;
202 EXPORT_SYMBOL_GPL(tegra_bwmgr_get_max_emc_rate);
204 /* Returns the ratio between dram and emc freq based on the type of dram */
205 int bwmgr_get_emc_to_dram_freq_factor(void)
207 return emc_to_dram_freq_factor;
209 EXPORT_SYMBOL_GPL(bwmgr_get_emc_to_dram_freq_factor);
211 /* Returns the actual emc frequency calculated using the dram
212 * frequency and emc_to_dram conversion factor
214 unsigned long tegra_bwmgr_get_core_emc_rate(void)
216 return (unsigned long)(tegra_bwmgr_get_emc_rate() /
217 bwmgr_get_emc_to_dram_freq_factor());
219 EXPORT_SYMBOL_GPL(tegra_bwmgr_get_core_emc_rate);
221 unsigned long tegra_bwmgr_round_rate(unsigned long bw)
224 return clk_round_rate(bwmgr.emc_clk, bw);
228 EXPORT_SYMBOL_GPL(tegra_bwmgr_round_rate);
230 int tegra_bwmgr_set_emc(struct tegra_bwmgr_client *handle, unsigned long val,
231 enum tegra_bwmgr_request_type req)
234 bool update_clk = false;
242 if (!IS_HANDLE_VALID(handle)) {
243 pr_err("bwmgr: client sent bad handle %p\n",
249 if (req >= TEGRA_BWMGR_SET_EMC_REQ_COUNT) {
250 pr_err("bwmgr: client %ld sent bad request type %d\n",
251 handle - bwmgr.bwmgr_client, req);
259 case TEGRA_BWMGR_SET_EMC_FLOOR:
260 if (handle->floor != val) {
266 case TEGRA_BWMGR_SET_EMC_CAP:
268 val = bwmgr.emc_max_rate;
270 if (handle->cap != val) {
276 case TEGRA_BWMGR_SET_EMC_ISO_CAP:
278 val = bwmgr.emc_max_rate;
280 if (handle->iso_cap != val) {
281 handle->iso_cap = val;
286 case TEGRA_BWMGR_SET_EMC_SHARED_BW:
287 if (handle->bw != val) {
293 case TEGRA_BWMGR_SET_EMC_SHARED_BW_ISO:
294 if (handle->iso_bw != val) {
295 handle->iso_bw = val;
305 if (update_clk && !clk_update_disabled)
306 ret = bwmgr_update_clk();
312 EXPORT_SYMBOL_GPL(tegra_bwmgr_set_emc);
314 int tegra_bwmgr_notifier_register(struct notifier_block *nb)
320 return clk_notifier_register(bwmgr.emc_clk, nb);
324 EXPORT_SYMBOL_GPL(tegra_bwmgr_notifier_register);
326 int tegra_bwmgr_notifier_unregister(struct notifier_block *nb)
332 return clk_notifier_unregister(bwmgr.emc_clk, nb);
336 EXPORT_SYMBOL_GPL(tegra_bwmgr_notifier_unregister);
338 /* Should be overrided always */
339 void __weak bwmgr_eff_init(void)
344 int get_iso_bw_table_idx(unsigned long iso_bw);
346 unsigned long tegra_bwmgr_get_emc_rate(void)
349 return clk_get_rate(bwmgr.emc_clk);
353 EXPORT_SYMBOL_GPL(tegra_bwmgr_get_emc_rate);
355 static unsigned long bwmgr_apply_efficiency(
356 unsigned long total_bw, unsigned long iso_bw,
357 unsigned long max_rate, u64 usage_flags,
358 unsigned long *iso_bw_min)
360 u8 efficiency = bwmgr_dram_efficiency;
361 if (total_bw && efficiency && (efficiency < 100)) {
362 total_bw = total_bw / efficiency;
363 total_bw = (total_bw < max_rate / 100) ?
364 (total_bw * 100) : max_rate;
367 efficiency = bwmgr_dram_iso_eff_table[get_iso_bw_table_idx(iso_bw)];
368 WARN_ON(efficiency == 1);
369 if (iso_bw && efficiency && (efficiency < 100)) {
370 iso_bw /= efficiency;
371 iso_bw = (iso_bw < max_rate / 100) ?
372 (iso_bw * 100) : max_rate;
376 *iso_bw_min = iso_bw;
378 return max(total_bw, iso_bw);
381 int bwmgr_iso_bw_percentage_max(void)
383 return bwmgr_iso_bw_percentage;
385 EXPORT_SYMBOL_GPL(bwmgr_iso_bw_percentage_max);
387 int __init bwmgr_init(void)
390 struct device_node *dn;
393 mutex_init(&bwmgr.lock);
394 bwmgr_debugfs_init();
398 dn = of_find_compatible_node(NULL, NULL, "nvidia,bwmgr");
400 pr_err("bwmgr: dt node not found.\n");
404 bwmgr.emc_clk = of_clk_get(dn, 0);
405 if (IS_ERR_OR_NULL(bwmgr.emc_clk)) {
406 pr_err("bwmgr: couldn't find emc clock.\n");
407 bwmgr.emc_clk = NULL;
412 round_rate = clk_round_rate(bwmgr.emc_clk, 0);
413 if (round_rate < 0) {
414 bwmgr.emc_min_rate = 0;
415 pr_err("bwmgr: couldn't get emc clock min rate.\n");
417 bwmgr.emc_min_rate = (unsigned long)round_rate;
419 /* Use LONG_MAX as downstream functions treats rate arg as signed */
420 round_rate = clk_round_rate(bwmgr.emc_clk, LONG_MAX);
421 if (round_rate < 0) {
422 bwmgr.emc_max_rate = 0;
423 pr_err("bwmgr: couldn't get emc clock max rate.\n");
425 bwmgr.emc_max_rate = (unsigned long)round_rate;
427 for (i = 0; i < TEGRA_BWMGR_CLIENT_COUNT; i++)
428 purge_client(bwmgr.bwmgr_client + i);
430 /* Check status property is okay or not. */
431 if (of_device_is_available(dn))
434 bwmgr.status = false;
437 subsys_initcall(bwmgr_init);
439 void __exit bwmgr_exit(void)
443 for (i = 0; i < TEGRA_BWMGR_CLIENT_COUNT; i++)
444 purge_client(bwmgr.bwmgr_client + i);
446 bwmgr.emc_clk = NULL;
447 mutex_destroy(&bwmgr.lock);
449 module_exit(bwmgr_exit);
451 #ifdef CONFIG_DEBUG_FS
452 static struct tegra_bwmgr_client *bwmgr_debugfs_client_handle;
453 static struct dentry *debugfs_dir;
454 static struct dentry *debugfs_node_floor;
455 static struct dentry *debugfs_node_cap;
456 static struct dentry *debugfs_node_iso_cap;
457 static struct dentry *debugfs_node_bw;
458 static struct dentry *debugfs_node_iso_bw;
459 static struct dentry *debugfs_node_emc_rate;
460 static struct dentry *debugfs_node_emc_min;
461 static struct dentry *debugfs_node_emc_max;
462 static struct dentry *debugfs_node_core_emc_rate;
463 static struct dentry *debugfs_node_clients_info;
465 /* keep in sync with tegra_bwmgr_client_id */
466 static const char * const tegra_bwmgr_client_names[] = {
508 static int bwmgr_debugfs_emc_rate_set(void *data, u64 val)
513 static int bwmgr_debugfs_emc_rate_get(void *data, u64 *val)
515 *val = tegra_bwmgr_get_emc_rate();
519 DEFINE_SIMPLE_ATTRIBUTE(fops_debugfs_emc_rate, bwmgr_debugfs_emc_rate_get,
520 bwmgr_debugfs_emc_rate_set, "%llu\n");
522 static int bwmgr_debugfs_core_emc_rate_get(void *data, u64 *val)
524 *val = tegra_bwmgr_get_core_emc_rate();
528 DEFINE_SIMPLE_ATTRIBUTE(fops_debugfs_core_emc_rate,
529 bwmgr_debugfs_core_emc_rate_get, NULL, "%llu\n");
531 static int bwmgr_debugfs_floor_set(void *data, u64 val)
533 tegra_bwmgr_set_emc(bwmgr_debugfs_client_handle, val,
534 TEGRA_BWMGR_SET_EMC_FLOOR);
538 static int bwmgr_debugfs_floor_get(void *data, u64 *val)
540 *val = bwmgr_debugfs_client_handle->floor;
544 DEFINE_SIMPLE_ATTRIBUTE(fops_debugfs_floor, bwmgr_debugfs_floor_get,
545 bwmgr_debugfs_floor_set, "%llu\n");
547 static int bwmgr_debugfs_cap_set(void *data, u64 val)
549 tegra_bwmgr_set_emc(bwmgr_debugfs_client_handle, val,
550 TEGRA_BWMGR_SET_EMC_CAP);
554 static int bwmgr_debugfs_cap_get(void *data, u64 *val)
556 *val = bwmgr_debugfs_client_handle->cap;
560 DEFINE_SIMPLE_ATTRIBUTE(fops_debugfs_cap, bwmgr_debugfs_cap_get,
561 bwmgr_debugfs_cap_set, "%llu\n");
563 static int bwmgr_debugfs_iso_cap_set(void *data, u64 val)
565 tegra_bwmgr_set_emc(bwmgr_debugfs_client_handle, val,
566 TEGRA_BWMGR_SET_EMC_ISO_CAP);
570 static int bwmgr_debugfs_iso_cap_get(void *data, u64 *val)
572 *val = bwmgr_debugfs_client_handle->iso_cap;
576 DEFINE_SIMPLE_ATTRIBUTE(fops_debugfs_iso_cap, bwmgr_debugfs_iso_cap_get,
577 bwmgr_debugfs_iso_cap_set, "%llu\n");
579 static int bwmgr_debugfs_bw_set(void *data, u64 val)
581 tegra_bwmgr_set_emc(bwmgr_debugfs_client_handle, val,
582 TEGRA_BWMGR_SET_EMC_SHARED_BW);
586 static int bwmgr_debugfs_bw_get(void *data, u64 *val)
588 *val = bwmgr_debugfs_client_handle->bw;
592 DEFINE_SIMPLE_ATTRIBUTE(fops_debugfs_bw, bwmgr_debugfs_bw_get,
593 bwmgr_debugfs_bw_set, "%llu\n");
595 static int bwmgr_debugfs_iso_bw_set(void *data, u64 val)
597 tegra_bwmgr_set_emc(bwmgr_debugfs_client_handle, val,
598 TEGRA_BWMGR_SET_EMC_SHARED_BW_ISO);
602 static int bwmgr_debugfs_iso_bw_get(void *data, u64 *val)
604 *val = bwmgr_debugfs_client_handle->iso_bw;
608 static int bwmgr_clients_info_show(struct seq_file *s, void *data)
613 seq_printf(s, "%15s%15s%15s%15s%15s%15s (Khz)\n", "Client",
614 "Floor", "SharedBw", "SharedIsoBw", "Cap",
616 for (i = 0; i < TEGRA_BWMGR_CLIENT_COUNT; i++) {
617 seq_printf(s, "%15s%15lu%15lu%15lu%15lu%15lu\n",
618 tegra_bwmgr_client_names[i],
619 bwmgr.bwmgr_client[i].floor / 1000,
620 bwmgr.bwmgr_client[i].bw / 1000,
621 bwmgr.bwmgr_client[i].iso_bw / 1000,
622 bwmgr.bwmgr_client[i].cap / 1000,
623 bwmgr.bwmgr_client[i].iso_cap / 1000);
625 seq_printf(s, "Total BW requested : %lu (Khz)\n",
626 debug_info.bw / 1000);
627 seq_printf(s, "Total ISO_BW requested : %lu (Khz)\n",
628 debug_info.iso_bw / 1000);
629 seq_printf(s, "Effective floor request : %lu (Khz)\n",
630 debug_info.floor / 1000);
631 seq_printf(s, "Effective NON_ISO_CAP : %lu (Khz)\n",
632 debug_info.non_iso_cap / 1000);
633 seq_printf(s, "Effective ISO_CAP : %lu (Khz)\n",
634 debug_info.iso_cap / 1000);
635 seq_printf(s, "Total BW + ISO_BW : %lu (Khz)\n",
636 (debug_info.bw + debug_info.iso_bw) / 1000);
637 seq_printf(s, "Total BW+ISO_BW after applying efficieny numbers: %lu (Khz)\n",
638 debug_info.total_bw_aftr_eff / 1000);
639 seq_printf(s, "Total ISO_BW after applying efficiency : %lu (Khz)\n",
640 debug_info.iso_bw_aftr_eff / 1000);
641 seq_printf(s, "EMC calculated rate : %lu (Khz)\n",
642 debug_info.calc_freq / 1000);
643 seq_printf(s, "EMC requested(rounded) rate : %lu (Khz)\n",
644 debug_info.req_freq / 1000);
645 seq_printf(s, "EMC current rate : %lu (Khz)\n",
646 tegra_bwmgr_get_emc_rate() / 1000);
650 DEFINE_SIMPLE_ATTRIBUTE(fops_debugfs_iso_bw, bwmgr_debugfs_iso_bw_get,
651 bwmgr_debugfs_iso_bw_set, "%llu\n");
653 static int bwmgr_clients_info_open(struct inode *inode, struct file *file)
655 return single_open(file, bwmgr_clients_info_show, inode->i_private);
658 static const struct file_operations fops_bwmgr_clients_info = {
659 .open = bwmgr_clients_info_open,
662 .release = single_release,
665 static void bwmgr_debugfs_init(void)
667 bwmgr_debugfs_client_handle =
668 tegra_bwmgr_register(TEGRA_BWMGR_CLIENT_DEBUG);
670 if (IS_ERR_OR_NULL(bwmgr_debugfs_client_handle)) {
671 bwmgr_debugfs_client_handle = NULL;
672 pr_err("bwmgr: could not register bwmgr debugfs client\n");
676 debugfs_dir = debugfs_create_dir("tegra_bwmgr", NULL);
679 "clk_update_disabled", S_IRWXU, debugfs_dir,
680 &clk_update_disabled);
681 debugfs_node_emc_min = debugfs_create_u64(
682 "emc_min_rate", S_IRUSR, debugfs_dir,
683 (u64 *) &bwmgr.emc_min_rate);
684 debugfs_node_emc_max = debugfs_create_u64(
685 "emc_max_rate", S_IRUSR, debugfs_dir,
686 (u64 *) &bwmgr.emc_max_rate);
687 debugfs_node_core_emc_rate = debugfs_create_file(
688 "core_emc_rate", S_IRUSR, debugfs_dir, NULL,
689 &fops_debugfs_core_emc_rate);
690 debugfs_node_emc_rate = debugfs_create_file
691 ("emc_rate", S_IRUSR, debugfs_dir, NULL,
692 &fops_debugfs_emc_rate);
693 debugfs_node_floor = debugfs_create_file
694 ("debug_client_floor", S_IRWXU, debugfs_dir, NULL,
695 &fops_debugfs_floor);
696 debugfs_node_cap = debugfs_create_file
697 ("debug_client_cap", S_IRWXU, debugfs_dir, NULL,
699 debugfs_node_iso_cap = debugfs_create_file
700 ("debug_client_iso_cap", S_IRWXU, debugfs_dir, NULL,
701 &fops_debugfs_iso_cap);
702 debugfs_node_bw = debugfs_create_file
703 ("debug_client_bw", S_IRWXU, debugfs_dir, NULL,
705 debugfs_node_iso_bw = debugfs_create_file
706 ("debug_client_iso_bw", S_IRWXU, debugfs_dir, NULL,
707 &fops_debugfs_iso_bw);
708 debugfs_node_clients_info = debugfs_create_file
709 ("bwmgr_clients_info", S_IRUGO, debugfs_dir, NULL,
710 &fops_bwmgr_clients_info);
712 pr_err("bwmgr: error creating bwmgr debugfs dir.\n");
715 debug_info.non_iso_cap = bwmgr.emc_max_rate;
716 debug_info.iso_cap = bwmgr.emc_max_rate;
721 static void bwmgr_debugfs_init(void) {};
722 #endif /* CONFIG_DEBUG_FS */