drivers/ata/ahci-tegra.c

   1 /*
   2  * ahci-tegra.c - AHCI SATA support for TEGRA AHCI device
   3  *
   4  * Copyright (c) 2011-2015, NVIDIA Corporation.  All rights reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  *
  19  * libata documentation is available via 'make {ps|pdf}docs',
  20  * as Documentation/DocBook/libata.*
  21  *
  22  * AHCI hardware documentation:
  23  * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
  24  * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
  25  *
  26  */
  27
  28 #include <linux/platform_device.h>
  29 #include <linux/irq.h>
  30 #include <linux/kernel.h>
  31 #include <linux/module.h>
  32 #include <linux/init.h>
  33 #include <linux/blkdev.h>
  34 #include <linux/delay.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/device.h>
  38 #include <linux/dmi.h>
  39 #include <linux/gpio.h>
  40 #include <linux/of_gpio.h>
  41 #include <scsi/scsi_host.h>
  42 #include <scsi/scsi_cmnd.h>
  43 #include <linux/libata.h>
  44 #include <linux/regulator/machine.h>
  45 #include <linux/pm_runtime.h>
  46 #include "ahci.h"
  47
  48 #include <linux/clk.h>
  49 #include <linux/clk/tegra.h>
  50 #include <linux/tegra-powergate.h>
  51 #include <linux/platform_data/tegra_ahci.h>
  52 #include <linux/tegra-soc.h>
  53
  54 #include "../../arch/arm/mach-tegra/iomap.h"
  55
  56 #define DRV_NAME        "tegra-sata"
  57 #define DRV_VERSION     "1.0"
  58
  59 #define ENABLE_AHCI_DBG_PRINT                   0
  60 #if ENABLE_AHCI_DBG_PRINT
  61 #define AHCI_DBG_PRINT(fmt, arg...)  printk(KERN_ERR fmt, ## arg)
  62 #else
  63 #define AHCI_DBG_PRINT(fmt, arg...) do {} while (0)
  64 #endif
  65
  66 /* number of AHCI ports */
  67 #define TEGRA_AHCI_NUM_PORTS                    1
  68
  69 /* idle timeout for PM in msec */
  70 #define TEGRA_AHCI_MIN_IDLE_TIME                1000
  71 #define TEGRA_AHCI_DEFAULT_IDLE_TIME            2000
  72
  73 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
  74 static u32 tegra_ahci_idle_time = TEGRA_AHCI_DEFAULT_IDLE_TIME;
  75 #endif
  76
  77 #define PORT_BASE       (TEGRA_SATA_BAR5_BASE + 0x100)
  78
  79 /* Bit 0 (EN_FPCI) to allow FPCI accesses to SATA */
  80 #define SATA_CONFIGURATION_0_OFFSET             0x180
  81 #define EN_FPCI                                 (1 << 0)
  82 #define CLK_OVERRIDE                            (1 << 31)
  83
  84 #define SATA_INTR_MASK_0_OFFSET                 0x188
  85 #define IP_INT_MASK                             (1 << 16)
  86
  87 /* Need to write 0x00400200 to 0x70020094 */
  88 #define SATA_FPCI_BAR5_0_OFFSET                 0x094
  89 #define PRI_ICTLR_CPU_IER_SET_0_OFFSET          0x024
  90 #define CPU_IER_SATA_CTL                        (1 << 23)
  91
  92 #define AHCI_BAR5_CONFIG_LOCATION               0x24
  93 #define TEGRA_SATA_BAR5_INIT_PROGRAM            0xFFFFFFFF
  94 #define TEGRA_SATA_BAR5_FINAL_PROGRAM           0x40020000
  95
  96 #define FUSE_SATA_CALIB_OFFSET                  0x224
  97 #define FUSE_SATA_CALIB_MASK                    0x3
  98
  99 #define T_SATA0_CFG_PHY_REG                     0x120
 100 #define T_SATA0_CFG_PHY_SQUELCH_MASK            (1 << 24)
 101 #define PHY_USE_7BIT_ALIGN_DET_FOR_SPD_MASK     (1 << 11)
 102
 103 #define T_SATA0_CFG_POWER_GATE                  0x4ac
 104 #define POWER_GATE_SSTS_RESTORED_MASK           (1 << 23)
 105 #define POWER_GATE_SSTS_RESTORED_YES            (1 << 23)
 106 #define POWER_GATE_SSTS_RESTORED_NO             (0 << 23)
 107
 108 #define T_SATA0_DBG0_OFFSET                     0x550
 109
 110 #define T_SATA0_INDEX_OFFSET                    0x680
 111 #define SATA0_NONE_SELECTED                     0
 112 #define SATA0_CH1_SELECTED                      (1 << 0)
 113
 114 #define T_SATA0_CHX_PHY_CTRL1_GEN1_OFFSET       0x690
 115 #define SATA0_CHX_PHY_CTRL1_GEN1_TX_AMP_SHIFT   0
 116 #define SATA0_CHX_PHY_CTRL1_GEN1_TX_AMP_MASK    (0xff << 0)
 117 #define SATA0_CHX_PHY_CTRL1_GEN1_TX_PEAK_SHIFT  8
 118 #define SATA0_CHX_PHY_CTRL1_GEN1_TX_PEAK_MASK   (0xff << 8)
 119
 120 #define T_SATA0_CHX_PHY_CTRL1_GEN2_OFFSET       0x694
 121 #define SATA0_CHX_PHY_CTRL1_GEN2_TX_AMP_SHIFT   0
 122 #define SATA0_CHX_PHY_CTRL1_GEN2_TX_AMP_MASK    (0xff << 0)
 123 #define SATA0_CHX_PHY_CTRL1_GEN2_TX_PEAK_SHIFT  12
 124 #define SATA0_CHX_PHY_CTRL1_GEN2_TX_PEAK_MASK   (0xff << 12)
 125 #define SATA0_CHX_PHY_CTRL1_GEN2_RX_EQ_SHIFT    24
 126 #define SATA0_CHX_PHY_CTRL1_GEN2_RX_EQ_MASK     (0xf << 24)
 127
 128 /* AHCI config space defines */
 129 #define TEGRA_PRIVATE_AHCI_CC_BKDR              0x4a4
 130 #define TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE     0x54c
 131 #define TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE_EN  (1 << 12)
 132 #define TEGRA_PRIVATE_AHCI_CC_BKDR_PGM          0x01060100
 133
 134 /* AHCI HBA_CAP */
 135 #define TEGRA_PRIVATE_AHCI_CAP_BKDR             0xa0
 136 #define T_SATA0_AHCI_HBA_CAP_BKDR               0x300
 137 #define AHCI_HBA_PLL_CTRL_0                     0xa8
 138
 139 #define CLAMP_TXCLK_ON_SLUMBER                  (1 << 13)
 140 #define CLAMP_TXCLK_ON_DEVSLP                   (1 << 24)
 141 #define NO_CLAMP_SHUT_DOWN                      (1 << 3)
 142
 143 #define TEGRA_SATA_IO_SPACE_OFFSET              4
 144 #define TEGRA_SATA_ENABLE_IO_SPACE              (1 << 0)
 145 #define TEGRA_SATA_ENABLE_MEM_SPACE             (1 << 1)
 146 #define TEGRA_SATA_ENABLE_BUS_MASTER            (1 << 2)
 147 #define TEGRA_SATA_ENABLE_SERR                  (1 << 8)
 148 #define TEGRA_SATA_CORE_CLOCK_FREQ_HZ           (105*1000*1000)
 149 #define TEGRA_SATA_OOB_CLOCK_FREQ_HZ            (272*1000*1000)
 150
 151 #define APB_PMC_SATA_PWRGT_0_REG                0x1ac
 152 #define APBDEV_PMC_PWRGATE_TOGGLE_0             0x30
 153 #define APBDEV_PMC_PWRGATE_STATUS_0             0x38
 154 #define APBDEV_PMC_REMOVE_CLAMPING_CMD_0        0x34
 155
 156 #define CLK_RST_SATA_PLL_CFG0_REG               0x490
 157 #define CLK_RST_SATA_PLL_CFG1_REG               0x494
 158 #define CLK_RST_CONTROLLER_RST_DEVICES_V_0      0x358
 159 #define CLK_RST_CONTROLLER_RST_DEVICES_W_0      0x35c
 160 #define CLK_RST_CONTROLLER_RST_DEV_W_CLR_0      0x43c
 161 #define CLK_RST_CONTROLLER_RST_DEV_V_CLR_0      0x434
 162 #define CLK_RST_CONTROLLER_CLK_ENB_V_CLR_0      0x444
 163 #define CLK_RST_CONTROLLER_CLK_ENB_V_SET_0      0x440
 164 #define CLK_RST_CONTROLLER_CLK_ENB_V_0          0x360
 165 #define CLK_RST_CONTROLLER_RST_DEV_W_SET        0x438
 166 #define CLK_RST_CONTROLLER_RST_DEV_V_SET        0x430
 167 #define SET_CEC_RESET                           0x100
 168
 169
 170 #define CLR_CLK_ENB_SATA_OOB                    (1 << 27)
 171 #define CLR_CLK_ENB_SATA                        (1 << 28)
 172
 173 #define CLK_RST_CONTROLLER_PLLE_MISC_0          0x0ec
 174 #define CLK_RST_CONTROLLER_PLLE_MISC_0_VALUE    0x00070300
 175 #define CLK_RST_CONTROLLER_PLLE_BASE_0          0xe8
 176 #define PLLE_ENABLE                             (1 << 30)
 177 #define CLK_RST_CONTROLLER_PLLE_AUX_0           0x48c
 178 #define CLK_RST_CONTROLLER_PLLE_AUX_0_MASK      (1 << 1)
 179
 180 #define CLR_SATACOLD_RST                        (1 << 1)
 181 #define SWR_SATACOLD_RST                        (1 << 1)
 182 #define SWR_SATA_RST                            (1 << 28)
 183 #define SWR_SATA_OOB_RST                        (1 << 27)
 184 #define DEVSLP_OVERRIDE                         (1 << 17)
 185 #define SDS_SUPPORT                             (1 << 13)
 186 #define DESO_SUPPORT                            (1 << 15)
 187 #define SATA_AUX_PAD_PLL_CNTL_1_REG             0x1100
 188 #define SATA_AUX_MISC_CNTL_1_REG                0x1108
 189 #define SATA_AUX_SPARE_CFG0_0                   0x1118
 190
 191 /* for APB_PMC_SATA_PWRGT_0_REG */
 192 #define PG_INFO_MASK                            (1 << 6)
 193 #define PG_INFO_ON                              (1 << 6)
 194 #define PG_INFO_OFF                             (0 << 6)
 195 #define PLLE_IDDQ_SWCTL_MASK                    (1 << 4)
 196 #define PADPHY_IDDQ_OVERRIDE_VALUE_MASK         (1 << 3)
 197 #define PADPHY_IDDQ_OVERRIDE_VALUE_ON           (1 << 3)
 198 #define PADPHY_IDDQ_OVERRIDE_VALUE_OFF          (0 << 3)
 199 #define PADPHY_IDDQ_SWCTL_MASK                  (1 << 2)
 200 #define PADPHY_IDDQ_SWCTL_ON                    (1 << 2)
 201 #define PADPHY_IDDQ_SWCTL_OFF                   (0 << 2)
 202 #define PADPLL_IDDQ_OVERRIDE_VALUE_MASK         (1 << 1)
 203 #define PADPLL_IDDQ_OVERRIDE_VALUE_ON           (1 << 1)
 204 #define PADPLL_IDDQ_OVERRIDE_VALUE_OFF          (0 << 1)
 205 #define PADPLL_IDDQ_SWCTL_MASK                  (1 << 0)
 206 #define PADPLL_IDDQ_SWCTL_ON                    (1 << 0)
 207 #define PADPLL_IDDQ_SWCTL_OFF                   (0 << 0)
 208
 209 #define START                                   (1 < 8)
 210 #define PARTID_VALUE                            0x8
 211
 212 #define SAX_MASK                                (1 << 8)
 213
 214 /* for CLK_RST_SATA_PLL_CFG0_REG */
 215 #define PADPLL_RESET_OVERRIDE_VALUE_MASK        (1 << 1)
 216 #define PADPLL_RESET_OVERRIDE_VALUE_ON          (1 << 1)
 217 #define PADPLL_RESET_OVERRIDE_VALUE_OFF         (0 << 1)
 218 #define PADPLL_RESET_SWCTL_MASK                 (1 << 0)
 219 #define PADPLL_RESET_SWCTL_ON                   (1 << 0)
 220 #define PADPLL_RESET_SWCTL_OFF                  (0 << 0)
 221 #define PLLE_IDDQ_SWCTL_ON                      (1 << 4)
 222 #define PLLE_IDDQ_SWCTL_OFF                     (0 << 4)
 223 #define PLLE_SATA_SEQ_ENABLE                    (1 << 24)
 224 #define PLLE_SATA_SEQ_START_STATE               (1 << 25)
 225 #define SATA_SEQ_PADPLL_PD_INPUT_VALUE          (1 << 5)
 226 #define SATA_SEQ_LANE_PD_INPUT_VALUE            (1 << 6)
 227 #define SATA_SEQ_RESET_INPUT_VALUE              (1 << 7)
 228
 229 /* for CLK_RST_SATA_PLL_CFG1_REG */
 230 #define IDDQ2LANE_SLUMBER_DLY_MASK              (0xffL << 16)
 231 #define IDDQ2LANE_SLUMBER_DLY_SHIFT             16
 232 #define IDDQ2LANE_SLUMBER_DLY_3MS               (3 << 16)
 233 #define IDDQ2LANE_IDDQ_DLY_SHIFT                0
 234 #define IDDQ2LANE_IDDQ_DLY_MASK                 (0xffL << 0)
 235
 236 /* for SATA_AUX_PAD_PLL_CNTL_1_REG */
 237 #define REFCLK_SEL_MASK                         (3 << 11)
 238 #define REFCLK_SEL_INT_CML                      (0 << 11)
 239 #define LOCKDET_FIELD                           (1 << 6)
 240
 241 /* for SATA_AUX_MISC_CNTL_1_REG */
 242 #define NVA2SATA_OOB_ON_POR_MASK                (1 << 7)
 243 #define NVA2SATA_OOB_ON_POR_YES                 (1 << 7)
 244 #define NVA2SATA_OOB_ON_POR_NO                  (0 << 7)
 245 #define L0_RX_IDLE_T_SAX_SHIFT                  5
 246 #define L0_RX_IDLE_T_SAX_MASK                   (3 << 5)
 247 #define L0_RX_IDLE_T_NPG_SHIFT                  3
 248 #define L0_RX_IDLE_T_NPG_MASK                   (3 << 3)
 249 #define L0_RX_IDLE_T_MUX_MASK                   (1 << 2)
 250 #define L0_RX_IDLE_T_MUX_FROM_APB_MISC          (1 << 2)
 251 #define L0_RX_IDLE_T_MUX_FROM_SATA              (0 << 2)
 252
 253 #define SSTAT_IPM_STATE_MASK                    0xF00
 254 #define SSTAT_IPM_SLUMBER_STATE                 0x600
 255 #define XUSB_PADCTL_USB3_PAD_MUX_0              0x134
 256 #define FORCE_SATA_PAD_IDDQ_DISABLE_MASK0       (1 << 6)
 257 #define XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0         0x138
 258 #define XUSB_PADCTL_PLL1_MODE                   (1 << 24)
 259 #define XUSB_PADCTL_ELPG_PROGRAM_0              0x01c
 260 #define AUX_ELPG_CLAMP_EN                       (1 << 24)
 261 #define AUX_ELPG_CLAMP_EN_EARLY                 (1 << 25)
 262 #define AUX_ELPG_VCORE_DOWN                     (1 << 26)
 263 #define XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0   0x148
 264 #define XUSB_PADCTL_IOPHY_MISC_IDDQ_OVRD        (1 << 1)
 265 #define XUSB_PADCTL_IOPHY_MISC_IDDQ             (1 << 0)
 266
 267 #define SATA_AXI_BAR5_START_0                   0x54
 268 #define SATA_AXI_BAR5_SZ_0                      0x14
 269 #define SATA_AXI_BAR5_START_VALUE               0x70020
 270 #define AXI_BAR5_SIZE_VALUE                     0x00008
 271 #define FPCI_BAR5_0_START_VALUE                 0x0010000
 272 #define FPCI_BAR5_0_FINAL_VALUE                 0x40020100
 273 #define FPCI_BAR5_0_ACCESS_TYPE                 (1 << 0)
 274
 275 #define XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0   0x148
 276 #define IDDQ_OVRD_MASK                          (1 << 1)
 277 #define IDDQ_MASK                               (1 << 0)
 278
 279 /* for XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0 */
 280 #define PLL_PWR_OVRD_MASK                       (1 << 3)
 281 #define PLL_RST_MASK                            (1 << 1)
 282 #define PLL_IDDQ_MASK                           (1 << 0)
 283
 284 /* for CLK_RST_CONTROLLER_PLLE_MISC_0 */
 285 #define T124_PLLE_IDDQ_SWCTL_MASK               (1 << 14)
 286 #define PLLE_IDDQ_OVERRIDE_VALUE_MASK           (1 << 13)
 287
 288 /* Spread Settings */
 289 #define SATA0_CHX_PHY_CTRL11_0                  0x6D0
 290 #define SATA0_CHX_PHY_CTRL2_0                   0x69c
 291 #define GEN2_RX_EQ                              (0x2800 << 16)
 292 #define CDR_CNTL_GEN1                           0x23
 293
 294 #define CLK_RST_CONTROLLER_PLLE_SS_CNTL_0       0x68
 295 #define PLLE_SSCCENTER                          (1 << 14)
 296 #define PLLE_SSCINVERT                          (1 << 15)
 297 #define PLLE_SSCMAX                             (0x25)
 298 #define PLLE_SSCINCINTRV                        (0x20 << 24)
 299 #define PLLE_SSCINC                             (1 << 16)
 300 #define PLLE_BYPASS_SS                          (1 << 10)
 301 #define PLLE_SSCBYP                             (1 << 12)
 302 #define PLLE_INTERP_RESET                       (1 << 11)
 303
 304 #define SATA_AUX_RX_STAT_INT_0                  0x110c
 305 #define SATA_RX_STAT_INT_DISABLE                (1 << 2)
 306
 307 #define T_SATA0_NVOOB                           0x114
 308 #define T_SATA0_NVOOB_SQUELCH_FILTER_MODE_SHIFT 24
 309 #define T_SATA0_NVOOB_SQUELCH_FILTER_MODE_MASK  (3 << 24)
 310 #define T_SATA0_NVOOB_SQUELCH_FILTER_LENGTH_SHIFT       26
 311 #define T_SATA0_NVOOB_SQUELCH_FILTER_LENGTH_MASK        (3 << 26)
 312
 313 #define T_SATA0_FIFO_0                          0x170
 314 #define T_SATA0_FIFO_0_L2P_FIFO_DEPTH           (0x7 << 12)
 315 #define T_SATA0_FIFO_0_L2P_FIFO_DEPTH_MASK      (0xf << 12)
 316
 317
 318 #define PXSSTS_DEVICE_DETECTED                  (1 << 0)
 319
 320 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
 321
 322 /* create a work for handling the async transfers */
 323 static void tegra_ahci_work_handler(struct work_struct *work);
 324 static DECLARE_WORK(tegra_ahci_work, tegra_ahci_work_handler);
 325 static struct workqueue_struct *tegra_ahci_work_q;
 326 #endif
 327
 328 enum {
 329         AHCI_PCI_BAR = 5,
 330 };
 331
 332 enum port_idle_status {
 333         PORT_IS_NOT_IDLE,
 334         PORT_IS_IDLE,
 335         PORT_IS_IDLE_NOT_SLUMBER,
 336         PORT_IS_SLUMBER,
 337 };
 338
 339 enum sata_state {
 340         SATA_ON,
 341         SATA_OFF,
 342         SATA_GOING_ON,
 343         SATA_GOING_OFF,
 344         SATA_ABORT_OFF,
 345 };
 346
 347 enum clk_gate_state {
 348         CLK_OFF,
 349         CLK_ON,
 350 };
 351
 352 enum sata_connectors {
 353         MINI_SATA,
 354         MICRO_SATA,
 355         SLIMLINE_SATA,
 356         E_SATA,
 357         E_SATA_P,
 358         SATA_EXPRESS,
 359         STANDARD_SATA,
 360 };
 361
 362 char *sata_power_rails[] = {
 363         "avdd_sata",
 364         "vdd_sata",
 365         "hvdd_sata",
 366         "avdd_sata_pll",
 367         "vddio_pex_sata"
 368 };
 369
 370 #define NUM_SATA_POWER_RAILS    ARRAY_SIZE(sata_power_rails)
 371
 372 struct tegra_qc_list {
 373         struct list_head list;
 374         struct ata_queued_cmd *qc;
 375 };
 376
 377 /*
 378  *  tegra_ahci_host_priv is the extension of ahci_host_priv
 379  *  with extra fields: idle_timer, pg_save, pg_state, etc.
 380  */
 381 struct tegra_ahci_host_priv {
 382         struct ahci_host_priv   ahci_host_priv;
 383         struct regulator        *power_rails[NUM_SATA_POWER_RAILS];
 384         void __iomem            *bars_table[6];
 385         struct ata_host         *host;
 386         struct timer_list       idle_timer;
 387         struct device           *dev;
 388         void                    *pg_save;
 389         enum sata_state         pg_state;
 390         enum sata_connectors    sata_connector;
 391         struct list_head        qc_list;
 392         struct clk              *clk_sata;
 393         struct clk              *clk_sata_oob;
 394         struct clk              *clk_pllp;
 395         struct clk              *clk_cml1;
 396         enum clk_gate_state     clk_state;
 397         s16                     gen2_rx_eq;
 398         int                     pexp_gpio;
 399 };
 400
 401 static int tegra_ahci_init_one(struct platform_device *pdev);
 402 static int tegra_ahci_remove_one(struct platform_device *pdev);
 403 static void tegra_ahci_set_clk_rst_cnt_rst_dev(void);
 404 static void tegra_ahci_clr_clk_rst_cnt_rst_dev(void);
 405 static void tegra_ahci_pad_config(void);
 406 static void tegra_ahci_put_sata_in_iddq(void);
 407 static void tegra_ahci_iddqlane_config(void);
 408
 409 #ifdef CONFIG_PM
 410 static bool tegra_ahci_power_un_gate(struct ata_host *host);
 411 static bool tegra_ahci_power_gate(struct ata_host *host);
 412 static void tegra_ahci_abort_power_gate(struct ata_host *host);
 413 static int tegra_ahci_controller_suspend(struct platform_device *pdev);
 414 static int tegra_ahci_controller_resume(struct platform_device *pdev);
 415 static int tegra_ahci_suspend(struct platform_device *pdev, pm_message_t mesg);
 416 static int tegra_ahci_resume(struct platform_device *pdev);
 417 static enum port_idle_status tegra_ahci_is_port_idle(struct ata_port *ap);
 418 static bool tegra_ahci_are_all_ports_idle(struct ata_host *host);
 419 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
 420 static bool tegra_ahci_pad_resume(struct ata_host *host);
 421 static bool tegra_ahci_pad_suspend(struct ata_host *host);
 422 static void tegra_ahci_abort_pad_suspend(struct ata_host *host);
 423 static unsigned int tegra_ahci_qc_issue(struct ata_queued_cmd *qc);
 424 static int tegra_ahci_hardreset(struct ata_link *link, unsigned int *class,
 425                                 unsigned long deadline);
 426 static int tegra_ahci_runtime_suspend(struct device *dev);
 427 static int tegra_ahci_runtime_resume(struct device *dev);
 428 static void tegra_ahci_idle_timer(unsigned long arg);
 429 static int tegra_ahci_queue_one_qc(struct tegra_ahci_host_priv *tegra_hpriv,
 430                                    struct ata_queued_cmd *qc);
 431 static void tegra_ahci_dequeue_qcs(struct tegra_ahci_host_priv *tegra_hpriv);
 432 #endif
 433 #else
 434 #define tegra_ahci_controller_suspend   NULL
 435 #define tegra_ahci_controller_resume    NULL
 436 #define tegra_ahci_suspend              NULL
 437 #define tegra_ahci_resume               NULL
 438 #endif
 439
 440 static struct scsi_host_template ahci_sht = {
 441         AHCI_SHT("tegra-sata"),
 442 };
 443
 444 static struct ata_port_operations tegra_ahci_ops = {
 445         .inherits       = &ahci_ops,
 446 #ifdef CONFIG_PM
 447 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
 448         .qc_issue       = tegra_ahci_qc_issue,
 449         .hardreset      = tegra_ahci_hardreset,
 450 #endif
 451 #endif
 452 };
 453
 454 static const struct ata_port_info ahci_port_info = {
 455         .flags          = AHCI_FLAG_COMMON,
 456         .pio_mask       = 0x1f, /* pio0-4 */
 457         .udma_mask      = ATA_UDMA6,
 458         .port_ops       = &tegra_ahci_ops,
 459 };
 460
 461 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
 462 static const struct dev_pm_ops tegra_ahci_dev_rt_ops = {
 463         .runtime_suspend = tegra_ahci_runtime_suspend,
 464         .runtime_resume = tegra_ahci_runtime_resume,
 465 };
 466 #endif
 467
 468 static const struct of_device_id of_ahci_tegra_match[] = {
 469         { .compatible = "nvidia,tegra114-ahci-sata", },
 470         {},
 471 };
 472 MODULE_DEVICE_TABLE(of, of_ahci_tegra_match);
 473
 474 static struct platform_driver tegra_platform_ahci_driver = {
 475         .probe          = tegra_ahci_init_one,
 476         .remove         = tegra_ahci_remove_one,
 477 #ifdef CONFIG_PM
 478         .suspend        = tegra_ahci_suspend,
 479         .resume         = tegra_ahci_resume,
 480 #endif
 481         .driver = {
 482                 .name = DRV_NAME,
 483                 .owner = THIS_MODULE,
 484                 .of_match_table = of_match_ptr(of_ahci_tegra_match),
 485         }
 486 };
 487
 488 struct tegra_ahci_host_priv *g_tegra_hpriv;
 489
 490 static inline u32 port_readl(u32 offset)
 491 {
 492         u32 val;
 493
 494         val = readl(IO_ADDRESS(PORT_BASE + offset));
 495         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", PORT_BASE + offset, val);
 496         return val;
 497 }
 498
 499 static inline void port_writel(u32 val, u32 offset)
 500 {
 501         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", PORT_BASE + offset, val);
 502         writel(val, IO_ADDRESS(PORT_BASE + offset));
 503         readl(IO_ADDRESS(PORT_BASE + offset));
 504 }
 505
 506 static inline u32 bar5_readl(u32 offset)
 507 {
 508         u32 val;
 509         val = readl(IO_ADDRESS(TEGRA_SATA_BAR5_BASE + offset));
 510         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_SATA_BAR5_BASE+offset,
 511                         val);
 512         return val;
 513 }
 514
 515 static inline void bar5_writel(u32 val, u32 offset)
 516 {
 517         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_SATA_BAR5_BASE+offset,
 518                         val);
 519         writel(val, IO_ADDRESS(TEGRA_SATA_BAR5_BASE + offset));
 520         readl(IO_ADDRESS(TEGRA_SATA_BAR5_BASE + offset));
 521 }
 522
 523
 524 static inline u32 xusb_readl(u32 offset)
 525 {
 526         u32 val;
 527         val = readl(IO_ADDRESS(TEGRA_XUSB_PADCTL_BASE + offset));
 528         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_XUSB_PADCTL_BASE+offset,
 529                         val);
 530         return val;
 531 }
 532
 533 static inline void xusb_writel(u32 val, u32 offset)
 534 {
 535         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_XUSB_PADCTL_BASE+offset,
 536                         val);
 537         writel(val, IO_ADDRESS(TEGRA_XUSB_PADCTL_BASE + offset));
 538         readl(IO_ADDRESS(TEGRA_XUSB_PADCTL_BASE + offset));
 539 }
 540
 541
 542 static inline u32 pmc_readl(u32 offset)
 543 {
 544         u32 val;
 545         val = readl(IO_ADDRESS(TEGRA_PMC_BASE + offset));
 546         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_PMC_BASE+offset, val);
 547         return val;
 548 }
 549
 550 static inline void pmc_writel(u32 val, u32 offset)
 551 {
 552         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_PMC_BASE+offset, val);
 553         writel(val, IO_ADDRESS(TEGRA_PMC_BASE + offset));
 554         readl(IO_ADDRESS(TEGRA_PMC_BASE + offset));
 555 }
 556
 557 static inline u32 clk_readl(u32 offset)
 558 {
 559         u32 val;
 560
 561         val = readl(IO_ADDRESS(TEGRA_CLK_RESET_BASE + offset));
 562         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_CLK_RESET_BASE+offset, val);
 563         return val;
 564 }
 565
 566 static inline void clk_writel(u32 val, u32 offset)
 567 {
 568         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_CLK_RESET_BASE+offset, val);
 569         writel(val, IO_ADDRESS(TEGRA_CLK_RESET_BASE + offset));
 570         readl(IO_ADDRESS(TEGRA_CLK_RESET_BASE + offset));
 571 }
 572
 573 static inline u32 misc_readl(u32 offset)
 574 {
 575         u32 val;
 576
 577         val = readl(IO_ADDRESS(TEGRA_APB_MISC_BASE + offset));
 578         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_APB_MISC_BASE+offset, val);
 579         return val;
 580 }
 581
 582 static inline void misc_writel(u32 val, u32 offset)
 583 {
 584         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_APB_MISC_BASE+offset, val);
 585         writel(val, IO_ADDRESS(TEGRA_APB_MISC_BASE + offset));
 586         readl(IO_ADDRESS(TEGRA_APB_MISC_BASE + offset));
 587 }
 588
 589 static inline u32 sata_readl(u32 offset)
 590 {
 591         u32 val;
 592
 593         val = readl(IO_ADDRESS(TEGRA_SATA_BASE + offset));
 594         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_SATA_BASE+offset, val);
 595         return val;
 596 }
 597
 598 static inline void sata_writel(u32 val, u32 offset)
 599 {
 600         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_SATA_BASE+offset, val);
 601         writel(val, IO_ADDRESS(TEGRA_SATA_BASE + offset));
 602         readl(IO_ADDRESS(TEGRA_SATA_BASE + offset));
 603 }
 604
 605 static inline u32 scfg_readl(u32 offset)
 606 {
 607         u32 val;
 608
 609         val = readl(IO_ADDRESS(TEGRA_SATA_CONFIG_BASE + offset));
 610         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_SATA_CONFIG_BASE+offset,
 611                                              val);
 612         return val;
 613 }
 614
 615 static inline void scfg_writel(u32 val, u32 offset)
 616 {
 617         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_SATA_CONFIG_BASE+offset,
 618                                              val);
 619         writel(val, IO_ADDRESS(TEGRA_SATA_CONFIG_BASE + offset));
 620         readl(IO_ADDRESS(TEGRA_SATA_CONFIG_BASE + offset));
 621 }
 622
 623 static inline u32 pictlr_readl(u32 offset)
 624 {
 625         u32 val;
 626
 627         val = readl(IO_ADDRESS(TEGRA_PRIMARY_ICTLR_BASE + offset));
 628         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_PRIMARY_ICTLR_BASE+offset,
 629                                              val);
 630         return val;
 631 }
 632
 633 static inline void pictlr_writel(u32 val, u32 offset)
 634 {
 635         AHCI_DBG_PRINT("[0x%x] <= 0x%08x\n", TEGRA_PRIMARY_ICTLR_BASE+offset,
 636                                              val);
 637         writel(val, IO_ADDRESS(TEGRA_PRIMARY_ICTLR_BASE + offset));
 638         readl(IO_ADDRESS(TEGRA_PRIMARY_ICTLR_BASE + offset));
 639 }
 640
 641 static inline u32 fuse_readl(u32 offset)
 642 {
 643         u32 val;
 644
 645         val = readl(IO_ADDRESS(TEGRA_FUSE_BASE + offset));
 646         AHCI_DBG_PRINT("[0x%x] => 0x%08x\n", TEGRA_FUSE_BASE+offset, val);
 647
 648         return val;
 649 }
 650
 651 /* Sata Pad Cntrl Values */
 652 struct sata_pad_cntrl {
 653         u8 gen1_tx_amp;
 654         u8 gen1_tx_peak;
 655         u8 gen2_tx_amp;
 656         u8 gen2_tx_peak;
 657 };
 658
 659 static const struct sata_pad_cntrl sata_calib_pad_val[] = {
 660         {       /* SATA_CALIB[1:0]  = 00 */
 661                 0x18,
 662                 0x04,
 663                 0x18,
 664                 0x0a
 665         },
 666         {       /* SATA_CALIB[1:0]  = 01 */
 667                 0x0e,
 668                 0x04,
 669                 0x14,
 670                 0x0a
 671         },
 672         {       /* SATA_CALIB[1:0]  = 10 */
 673                 0x0e,
 674                 0x07,
 675                 0x1a,
 676                 0x0e
 677         },
 678         {       /* SATA_CALIB[1:0]  = 11 */
 679                 0x14,
 680                 0x0e,
 681                 0x1a,
 682                 0x0e
 683         }
 684 };
 685
 686 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
 687 static void tegra_ahci_work_handler(struct work_struct *work)
 688 {
 689         tegra_ahci_runtime_resume(g_tegra_hpriv->dev);
 690 }
 691 #endif
 692
 693 static void tegra_ahci_set_pad_cntrl_regs(
 694                         struct tegra_ahci_host_priv *tegra_hpriv)
 695 {
 696         int     calib_val;
 697         int     val;
 698         int     i;
 699
 700         calib_val = fuse_readl(FUSE_SATA_CALIB_OFFSET) & FUSE_SATA_CALIB_MASK;
 701
 702         val = clk_readl(CLK_RST_CONTROLLER_PLLE_SS_CNTL_0);
 703         val &= ~(PLLE_SSCCENTER | PLLE_SSCINVERT);
 704         val |= (PLLE_SSCMAX | PLLE_SSCINCINTRV | PLLE_SSCINC);
 705         clk_writel(val, CLK_RST_CONTROLLER_PLLE_SS_CNTL_0);
 706
 707         val = clk_readl(CLK_RST_CONTROLLER_PLLE_SS_CNTL_0);
 708         val &= ~(PLLE_BYPASS_SS | PLLE_SSCBYP);
 709         clk_writel(val, CLK_RST_CONTROLLER_PLLE_SS_CNTL_0);
 710
 711         udelay(2);
 712
 713         val = clk_readl(CLK_RST_CONTROLLER_PLLE_SS_CNTL_0);
 714         val &= ~(PLLE_INTERP_RESET);
 715         clk_writel(val, CLK_RST_CONTROLLER_PLLE_SS_CNTL_0);
 716
 717         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i) {
 718                 scfg_writel((1 << i), T_SATA0_INDEX_OFFSET);
 719
 720                 val = scfg_readl(T_SATA0_CHX_PHY_CTRL1_GEN1_OFFSET);
 721                 val &= ~SATA0_CHX_PHY_CTRL1_GEN1_TX_AMP_MASK;
 722                 val |= (sata_calib_pad_val[calib_val].gen1_tx_amp <<
 723                         SATA0_CHX_PHY_CTRL1_GEN1_TX_AMP_SHIFT);
 724                 scfg_writel(val, T_SATA0_CHX_PHY_CTRL1_GEN1_OFFSET);
 725
 726                 val = scfg_readl(T_SATA0_CHX_PHY_CTRL1_GEN1_OFFSET);
 727                 val &= ~SATA0_CHX_PHY_CTRL1_GEN1_TX_PEAK_MASK;
 728                 val |= (sata_calib_pad_val[calib_val].gen1_tx_peak <<
 729                         SATA0_CHX_PHY_CTRL1_GEN1_TX_PEAK_SHIFT);
 730                 scfg_writel(val, T_SATA0_CHX_PHY_CTRL1_GEN1_OFFSET);
 731
 732                 val = scfg_readl(T_SATA0_CHX_PHY_CTRL1_GEN2_OFFSET);
 733                 val &= ~SATA0_CHX_PHY_CTRL1_GEN2_TX_AMP_MASK;
 734                 val |= (sata_calib_pad_val[calib_val].gen2_tx_amp <<
 735                         SATA0_CHX_PHY_CTRL1_GEN2_TX_AMP_SHIFT);
 736                 scfg_writel(val, T_SATA0_CHX_PHY_CTRL1_GEN2_OFFSET);
 737
 738                 val = scfg_readl(T_SATA0_CHX_PHY_CTRL1_GEN2_OFFSET);
 739                 val &= ~SATA0_CHX_PHY_CTRL1_GEN2_TX_PEAK_MASK;
 740                 val |= (sata_calib_pad_val[calib_val].gen2_tx_peak <<
 741                         SATA0_CHX_PHY_CTRL1_GEN2_TX_PEAK_SHIFT);
 742                 scfg_writel(val, T_SATA0_CHX_PHY_CTRL1_GEN2_OFFSET);
 743
 744                 val = GEN2_RX_EQ;
 745                 scfg_writel(val, SATA0_CHX_PHY_CTRL11_0);
 746
 747                 val = CDR_CNTL_GEN1;
 748                 scfg_writel(val, SATA0_CHX_PHY_CTRL2_0);
 749         }
 750         scfg_writel(SATA0_NONE_SELECTED, T_SATA0_INDEX_OFFSET);
 751 }
 752
 753 int tegra_ahci_get_rails(struct regulator *regulators[])
 754 {
 755         struct regulator *reg;
 756         int i;
 757         int ret = 0;
 758
 759         for (i = 0; i < NUM_SATA_POWER_RAILS; ++i) {
 760                 reg = regulator_get(g_tegra_hpriv->dev, sata_power_rails[i]);
 761                 if (IS_ERR_OR_NULL(reg)) {
 762                         pr_err("%s: can't get regulator %s\n",
 763                                 __func__, sata_power_rails[i]);
 764                         WARN_ON(1);
 765                         ret = PTR_ERR(reg);
 766                         goto exit;
 767                 }
 768                 regulators[i] = reg;
 769         }
 770 exit:
 771         return ret;
 772 }
 773
 774 void tegra_ahci_put_rails(struct regulator *regulators[])
 775 {
 776         int i;
 777
 778         for (i = 0; i < NUM_SATA_POWER_RAILS; ++i)
 779                 regulator_put(regulators[i]);
 780 }
 781
 782 int tegra_ahci_power_on_rails(struct regulator *regulators[])
 783 {
 784         struct regulator *reg;
 785         int i;
 786         int ret = 0;
 787
 788         for (i = 0; i < NUM_SATA_POWER_RAILS; ++i) {
 789                 reg = regulators[i];
 790                 ret = regulator_enable(reg);
 791                 if (ret) {
 792                         pr_err("%s: can't enable regulator[%d]\n",
 793                                 __func__, i);
 794                         WARN_ON(1);
 795                         goto exit;
 796                 }
 797         }
 798
 799 exit:
 800         return ret;
 801 }
 802
 803 int tegra_ahci_power_off_rails(struct regulator *regulators[])
 804 {
 805         struct regulator *reg;
 806         int i;
 807         int ret = 0;
 808
 809         for (i = 0; i < NUM_SATA_POWER_RAILS; ++i) {
 810                 reg = regulators[i];
 811                 if (!IS_ERR_OR_NULL(reg)) {
 812                         ret = regulator_disable(reg);
 813                         if (ret) {
 814                                 pr_err("%s: can't disable regulator[%d]\n",
 815                                         __func__, i);
 816                                 WARN_ON(1);
 817                                 goto exit;
 818                         }
 819                 }
 820         }
 821
 822 exit:
 823         return ret;
 824 }
 825
 826 static void tegra_first_level_clk_gate(void)
 827 {
 828         if (g_tegra_hpriv->clk_state == CLK_OFF)
 829                 return;
 830
 831         clk_disable_unprepare(g_tegra_hpriv->clk_sata);
 832         clk_disable_unprepare(g_tegra_hpriv->clk_sata_oob);
 833         clk_disable_unprepare(g_tegra_hpriv->clk_cml1);
 834         g_tegra_hpriv->clk_state = CLK_OFF;
 835 }
 836
 837 static int tegra_first_level_clk_ungate(void)
 838 {
 839         int ret = 0;
 840
 841         if (g_tegra_hpriv->clk_state == CLK_ON) {
 842                 ret = -1;
 843                 return ret;
 844         }
 845
 846         if (clk_prepare_enable(g_tegra_hpriv->clk_sata)) {
 847                 pr_err("%s: unable to enable SATA clock\n", __func__);
 848                 ret = -ENODEV;
 849                 return ret;
 850         }
 851         if (clk_prepare_enable(g_tegra_hpriv->clk_sata_oob)) {
 852                 pr_err("%s: unable to enable SATA_OOB clock\n", __func__);
 853                 clk_disable_unprepare(g_tegra_hpriv->clk_sata);
 854                 ret = -ENODEV;
 855                 return ret;
 856         }
 857         if (clk_prepare_enable(g_tegra_hpriv->clk_cml1)) {
 858                 pr_err("%s: unable to enable cml1 clock\n", __func__);
 859                 clk_disable_unprepare(g_tegra_hpriv->clk_sata);
 860                 clk_disable_unprepare(g_tegra_hpriv->clk_sata_oob);
 861                 ret = -ENODEV;
 862                 return ret;
 863         }
 864         g_tegra_hpriv->clk_state = CLK_ON;
 865
 866         return ret;
 867 }
 868
 869 static int tegra_ahci_controller_init(struct tegra_ahci_host_priv *tegra_hpriv,
 870                 int lp0)
 871 {
 872         int err;
 873         struct clk *clk_sata = NULL;
 874         struct clk *clk_sata_oob = NULL;
 875         struct clk *clk_sata_cold = NULL;
 876         struct clk *clk_pllp = NULL;
 877         struct clk *clk_cml1 = NULL;
 878         u32 val;
 879         u32 timeout;
 880
 881         if (!lp0) {
 882                 err = tegra_ahci_get_rails(tegra_hpriv->power_rails);
 883                 if (err) {
 884                         pr_err("%s: fails to get rails (%d)\n", __func__, err);
 885                         goto exit;
 886                 }
 887
 888         }
 889         err = tegra_ahci_power_on_rails(tegra_hpriv->power_rails);
 890         if (err) {
 891                 pr_err("%s: fails to power on rails (%d)\n", __func__, err);
 892                 goto exit;
 893         }
 894
 895         /* pll_p is the parent of tegra_sata and tegra_sata_oob */
 896         clk_pllp = clk_get_sys(NULL, "pll_p");
 897         if (IS_ERR_OR_NULL(clk_pllp)) {
 898                 pr_err("%s: unable to get PLL_P clock\n", __func__);
 899                 err = PTR_ERR(clk_pllp);
 900                 goto exit;
 901         }
 902
 903         clk_sata = clk_get_sys("tegra_sata", NULL);
 904         if (IS_ERR_OR_NULL(clk_sata)) {
 905                 pr_err("%s: unable to get SATA clock\n", __func__);
 906                 err = PTR_ERR(clk_sata);
 907                 goto exit;
 908         }
 909
 910         clk_sata_oob = clk_get_sys("tegra_sata_oob", NULL);
 911         if (IS_ERR_OR_NULL(clk_sata_oob)) {
 912                 pr_err("%s: unable to get SATA OOB clock\n", __func__);
 913                 err = PTR_ERR(clk_sata_oob);
 914                 goto exit;
 915         }
 916
 917         clk_sata_cold = clk_get_sys("tegra_sata_cold", NULL);
 918         if (IS_ERR_OR_NULL(clk_sata_cold)) {
 919                 pr_err("%s: unable to get SATA COLD clock\n", __func__);
 920                 err = PTR_ERR(clk_sata_cold);
 921                 goto exit;
 922         }
 923
 924         tegra_hpriv->clk_sata = clk_sata;
 925         tegra_hpriv->clk_sata_oob = clk_sata_oob;
 926         tegra_hpriv->clk_pllp = clk_pllp;
 927
 928         tegra_periph_reset_assert(clk_sata);
 929         tegra_periph_reset_assert(clk_sata_oob);
 930         tegra_periph_reset_assert(clk_sata_cold);
 931         udelay(10);
 932
 933         /* need to establish both clocks divisors before setting clk sources */
 934         clk_set_rate(clk_sata, clk_get_rate(clk_sata)/10);
 935         clk_set_rate(clk_sata_oob, clk_get_rate(clk_sata_oob)/10);
 936
 937         /* set SATA clk and SATA_OOB clk source */
 938         clk_set_parent(clk_sata, clk_pllp);
 939         clk_set_parent(clk_sata_oob, clk_pllp);
 940
 941         /* Configure SATA clocks */
 942         /* Core clock runs at 108MHz */
 943         if (clk_set_rate(clk_sata, TEGRA_SATA_CORE_CLOCK_FREQ_HZ)) {
 944                 err = -ENODEV;
 945                 goto exit;
 946         }
 947         /* OOB clock runs at 216MHz */
 948         if (clk_set_rate(clk_sata_oob, TEGRA_SATA_OOB_CLOCK_FREQ_HZ)) {
 949                 err = -ENODEV;
 950                 goto exit;
 951         }
 952
 953         if (clk_prepare_enable(clk_sata)) {
 954                 pr_err("%s: unable to enable SATA clock\n", __func__);
 955                 err = -ENODEV;
 956                 goto exit;
 957         }
 958
 959         if (clk_prepare_enable(clk_sata_oob)) {
 960                 pr_err("%s: unable to enable SATA OOB clock\n", __func__);
 961                 err = -ENODEV;
 962                 goto exit;
 963         }
 964
 965         tegra_periph_reset_deassert(clk_sata);
 966         tegra_periph_reset_deassert(clk_sata_oob);
 967         tegra_periph_reset_deassert(clk_sata_cold);
 968
 969         tegra_ahci_clr_clk_rst_cnt_rst_dev();
 970
 971         if (gpio_is_valid(tegra_hpriv->pexp_gpio)) {
 972                 val = gpio_request(tegra_hpriv->pexp_gpio,
 973                                 "ahci-tegra");
 974                 if (val) {
 975                         pr_err("failed to allocate Port expander gpio\n");
 976                         err = -ENODEV;
 977                         goto exit;
 978                 }
 979                 gpio_direction_output(tegra_hpriv->pexp_gpio, 1);
 980         }
 981
 982         val = 0x100;
 983         pmc_writel(val, APBDEV_PMC_REMOVE_CLAMPING_CMD_0);
 984
 985
 986         /* SATA_PADPLL_RESET_OVERRIDE_VALUE=1 and SATA_PADPLL_RESET_SWCTL=1 */
 987         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
 988         val |= (PADPLL_RESET_OVERRIDE_VALUE_ON | PADPLL_RESET_SWCTL_ON);
 989         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
 990
 991         /* select internal CML ref clk
 992          * select PLLE as input to IO phy */
 993         val = misc_readl(SATA_AUX_PAD_PLL_CNTL_1_REG);
 994         val &= ~REFCLK_SEL_MASK;
 995         val |= REFCLK_SEL_INT_CML;
 996         misc_writel(val, SATA_AUX_PAD_PLL_CNTL_1_REG);
 997
 998         /* wait for SATA_PADPLL_IDDQ2LANE_SLUMBER_DLY = 3 microseconds. */
 999         val = clk_readl(CLK_RST_SATA_PLL_CFG1_REG);
1000         val &= ~IDDQ2LANE_SLUMBER_DLY_MASK;
1001         val |= IDDQ2LANE_SLUMBER_DLY_3MS;
1002         clk_writel(val, CLK_RST_SATA_PLL_CFG1_REG);
1003         udelay(3);
1004
1005 #if defined(CONFIG_TEGRA_SILICON_PLATFORM)
1006         err = tegra_unpowergate_partition_with_clk_on(TEGRA_POWERGATE_SATA);
1007         if (err) {
1008                 pr_err("%s: ** failed to turn-on SATA (0x%x) **\n",
1009                                 __func__, err);
1010                 goto exit;
1011         }
1012 #endif
1013
1014         /*
1015          * place SATA Pad PLL out of reset by writing
1016          * SATA_PADPLL_RST_OVERRIDE_VALUE = 0
1017          */
1018         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
1019         val &= ~PADPLL_RESET_OVERRIDE_VALUE_MASK;
1020         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
1021
1022         /*
1023          * Wait for SATA_AUX_PAD_PLL_CNTL_1_0_LOCKDET to turn 1 with a timeout
1024          * of 15 us.
1025          */
1026         timeout = 15;
1027         while (timeout--) {
1028                 udelay(1);
1029                 val = xusb_readl(XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
1030                 if (val & LOCKDET_FIELD)
1031                         break;
1032         }
1033         if (timeout == 0)
1034                 pr_err("%s: AUX_PAD_PLL_CNTL_1 (0x%x) is not locked in 15us.\n",
1035                         __func__, val);
1036
1037         tegra_ahci_pad_config();
1038
1039         clk_cml1 = clk_get_sys(NULL, "cml1");
1040         if (IS_ERR_OR_NULL(clk_cml1)) {
1041                 pr_err("%s: unable to get cml1 clock Errone is %d\n",
1042                                         __func__, (int) PTR_ERR(clk_cml1));
1043                 err = PTR_ERR(clk_cml1);
1044                 goto exit;
1045         }
1046         tegra_hpriv->clk_cml1 = clk_cml1;
1047         if (clk_prepare_enable(clk_cml1)) {
1048                 pr_err("%s: unable to enable cml1 clock\n", __func__);
1049                 err = -ENODEV;
1050                 goto exit;
1051         }
1052
1053         g_tegra_hpriv->clk_state = CLK_ON;
1054
1055         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
1056         val &= ~PADPLL_RESET_SWCTL_MASK;
1057         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
1058
1059         /* PLLE Programing for SATA */
1060
1061         val = clk_readl(CLK_RST_CONTROLLER_PLLE_AUX_0);
1062         val |= CLK_RST_CONTROLLER_PLLE_AUX_0_MASK;
1063         clk_writel(val, CLK_RST_CONTROLLER_PLLE_AUX_0);
1064
1065         /* bring SATA IOPHY out of IDDQ */
1066         val = xusb_readl(XUSB_PADCTL_USB3_PAD_MUX_0);
1067         val |= FORCE_SATA_PAD_IDDQ_DISABLE_MASK0;
1068         xusb_writel(val, XUSB_PADCTL_USB3_PAD_MUX_0);
1069
1070         val = xusb_readl(XUSB_PADCTL_ELPG_PROGRAM_0);
1071         val &= ~(AUX_ELPG_CLAMP_EN | AUX_ELPG_CLAMP_EN_EARLY |
1072                 AUX_ELPG_VCORE_DOWN);
1073         xusb_writel(val, XUSB_PADCTL_ELPG_PROGRAM_0);
1074
1075         val = xusb_readl(XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
1076         val = val | XUSB_PADCTL_PLL1_MODE;
1077         xusb_writel(val, XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
1078
1079         /* clear NVA2SATA_OOB_ON_POR in SATA_AUX_MISC_CNTL_1_REG */
1080         val = misc_readl(SATA_AUX_MISC_CNTL_1_REG);
1081         val &= ~NVA2SATA_OOB_ON_POR_MASK;
1082         misc_writel(val, SATA_AUX_MISC_CNTL_1_REG);
1083
1084         if (tegra_hpriv->sata_connector != MINI_SATA) {
1085                 /* Disable DEVSLP Feature */
1086                 val = misc_readl(SATA_AUX_MISC_CNTL_1_REG);
1087                 val &= ~SDS_SUPPORT;
1088                 misc_writel(val, SATA_AUX_MISC_CNTL_1_REG);
1089         }
1090
1091         val = sata_readl(SATA_CONFIGURATION_0_OFFSET);
1092         val |= EN_FPCI;
1093         sata_writel(val, SATA_CONFIGURATION_0_OFFSET);
1094
1095         val = sata_readl(SATA_CONFIGURATION_0_OFFSET);
1096         val |= CLK_OVERRIDE;
1097         sata_writel(val, SATA_CONFIGURATION_0_OFFSET);
1098
1099         /* program sata pad control based on the fuse */
1100         tegra_ahci_set_pad_cntrl_regs(tegra_hpriv);
1101
1102         /*
1103          * clear bit T_SATA0_CFG_PHY_0_USE_7BIT_ALIGN_DET_FOR_SPD of
1104          * T_SATA0_CFG_PHY_0
1105          */
1106         val = scfg_readl(T_SATA0_CFG_PHY_REG);
1107         val |= T_SATA0_CFG_PHY_SQUELCH_MASK;
1108         scfg_writel(val, T_SATA0_CFG_PHY_REG);
1109
1110         val = scfg_readl(T_SATA0_NVOOB);
1111         val |= (1 << T_SATA0_NVOOB_SQUELCH_FILTER_MODE_SHIFT);
1112         val |= (3 << T_SATA0_NVOOB_SQUELCH_FILTER_LENGTH_SHIFT);
1113         scfg_writel(val, T_SATA0_NVOOB);
1114
1115         val = scfg_readl(T_SATA0_FIFO_0);
1116         val = (val & ~T_SATA0_FIFO_0_L2P_FIFO_DEPTH_MASK) |
1117                         T_SATA0_FIFO_0_L2P_FIFO_DEPTH;
1118         scfg_writel(val, T_SATA0_FIFO_0);
1119
1120         /*
1121          * WAR: Before enabling SATA PLL shutdown, lockdet needs to be ignored.
1122          *      To ignore lockdet, T_SATA0_DBG0_OFFSET register bit 10 needs to
1123          *      be 1, and bit 8 needs to be 0.
1124          */
1125         val = scfg_readl(T_SATA0_DBG0_OFFSET);
1126         val |= (1 << 10);
1127         val &= ~(1 << 8);
1128         scfg_writel(val, T_SATA0_DBG0_OFFSET);
1129
1130         /* program class code and programming interface for AHCI */
1131         val = scfg_readl(TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE);
1132         val |= TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE_EN;
1133         scfg_writel(val, TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE);
1134         scfg_writel(TEGRA_PRIVATE_AHCI_CC_BKDR_PGM, TEGRA_PRIVATE_AHCI_CC_BKDR);
1135         val &= ~TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE_EN;
1136         scfg_writel(val, TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE);
1137
1138         /* Program config space registers: */
1139
1140         /* Enable BUS_MASTER+MEM+IO space, and SERR */
1141         val = scfg_readl(TEGRA_SATA_IO_SPACE_OFFSET);
1142         val |= TEGRA_SATA_ENABLE_IO_SPACE | TEGRA_SATA_ENABLE_MEM_SPACE |
1143                TEGRA_SATA_ENABLE_BUS_MASTER | TEGRA_SATA_ENABLE_SERR;
1144         scfg_writel(val, TEGRA_SATA_IO_SPACE_OFFSET);
1145
1146         /* program bar5 space, by first writing 1's to bar5 register */
1147         scfg_writel(TEGRA_SATA_BAR5_INIT_PROGRAM, AHCI_BAR5_CONFIG_LOCATION);
1148         /* flush */
1149         val = scfg_readl(AHCI_BAR5_CONFIG_LOCATION);
1150
1151         /* then, write the BAR5_FINAL_PROGRAM address */
1152         scfg_writel(TEGRA_SATA_BAR5_FINAL_PROGRAM, AHCI_BAR5_CONFIG_LOCATION);
1153         /* flush */
1154         scfg_readl(AHCI_BAR5_CONFIG_LOCATION);
1155
1156         sata_writel((FPCI_BAR5_0_FINAL_VALUE >> 8),
1157                         SATA_FPCI_BAR5_0_OFFSET);
1158
1159         val = scfg_readl(T_SATA0_AHCI_HBA_CAP_BKDR);
1160         val |= (HOST_CAP_ALPM | HOST_CAP_SSC | HOST_CAP_PART | HOST_CAP_PMP);
1161         scfg_writel(val, T_SATA0_AHCI_HBA_CAP_BKDR);
1162
1163         /* Second Level Clock Gating*/
1164         val = bar5_readl(AHCI_HBA_PLL_CTRL_0);
1165         val |= (CLAMP_TXCLK_ON_SLUMBER | CLAMP_TXCLK_ON_DEVSLP);
1166         val &= ~NO_CLAMP_SHUT_DOWN;
1167         bar5_writel(val, AHCI_HBA_PLL_CTRL_0);
1168
1169         /* set IP_INT_MASK */
1170         val = sata_readl(SATA_INTR_MASK_0_OFFSET);
1171         val |= IP_INT_MASK;
1172         sata_writel(val, SATA_INTR_MASK_0_OFFSET);
1173
1174 exit:
1175         if (!IS_ERR_OR_NULL(clk_pllp))
1176                 clk_put(clk_pllp);
1177         if (!IS_ERR_OR_NULL(clk_sata))
1178                 clk_put(clk_sata);
1179         if (!IS_ERR_OR_NULL(clk_sata_oob))
1180                 clk_put(clk_sata_oob);
1181         if (!IS_ERR_OR_NULL(clk_sata_cold))
1182                 clk_put(clk_sata_cold);
1183         if (!IS_ERR_OR_NULL(clk_cml1))
1184                 clk_put(clk_cml1);
1185
1186         if (err) {
1187                 /* turn off all SATA power rails; ignore returned status */
1188                 tegra_ahci_power_off_rails(tegra_hpriv->power_rails);
1189                 /* return regulators to system */
1190                 tegra_ahci_put_rails(tegra_hpriv->power_rails);
1191         }
1192         return err;
1193 }
1194
1195 static void tegra_ahci_save_initial_config(struct platform_device *pdev,
1196                                            struct ahci_host_priv *hpriv)
1197 {
1198         ahci_save_initial_config(&pdev->dev, hpriv, 0, 0);
1199 }
1200
1201 static void tegra_ahci_controller_remove(struct platform_device *pdev)
1202 {
1203         struct ata_host *host = dev_get_drvdata(&pdev->dev);
1204         struct tegra_ahci_host_priv *tegra_hpriv;
1205         int status;
1206
1207         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1208
1209 #ifdef CONFIG_PM
1210         /* call tegra_ahci_controller_suspend() to power-down the SATA */
1211         status = tegra_ahci_controller_suspend(pdev);
1212         if (status)
1213                 dev_err(host->dev, "remove: error suspend SATA (0x%x)\n",
1214                                    status);
1215 #else
1216         /* power off the sata */
1217         status = tegra_powergate_partition_with_clk_off(TEGRA_POWERGATE_SATA);
1218         if (status)
1219                 dev_err(host->dev, "remove: error turn-off SATA (0x%x)\n",
1220                                    status);
1221         tegra_ahci_power_off_rails(tegra_hpriv->power_rails);
1222 #endif
1223
1224         /* return system resources */
1225         tegra_ahci_put_rails(tegra_hpriv->power_rails);
1226 }
1227
1228 #ifdef CONFIG_PM
1229 static int tegra_ahci_controller_suspend(struct platform_device *pdev)
1230 {
1231         struct ata_host *host = dev_get_drvdata(&pdev->dev);
1232         struct tegra_ahci_host_priv *tegra_hpriv;
1233         unsigned long flags;
1234
1235         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1236
1237         /* stop the idle timer */
1238         if (timer_pending(&tegra_hpriv->idle_timer))
1239                 del_timer_sync(&tegra_hpriv->idle_timer);
1240
1241         spin_lock_irqsave(&host->lock, flags);
1242         if (tegra_hpriv->pg_state == SATA_OFF)
1243                 dev_dbg(host->dev, "suspend: SATA already power gated\n");
1244         else {
1245                 bool pg_ok;
1246
1247                 dev_dbg(host->dev, "suspend: power gating SATA...\n");
1248                 pg_ok = tegra_ahci_power_gate(host);
1249                 if (pg_ok) {
1250                         tegra_hpriv->pg_state = SATA_OFF;
1251                         dev_dbg(host->dev, "suspend: SATA is power gated\n");
1252                 } else {
1253                         tegra_ahci_abort_power_gate(host);
1254                         spin_unlock_irqrestore(&host->lock, flags);
1255                         return -EBUSY;
1256                 }
1257         }
1258
1259         if (gpio_is_valid(tegra_hpriv->pexp_gpio))
1260                 gpio_free(tegra_hpriv->pexp_gpio);
1261         spin_unlock_irqrestore(&host->lock, flags);
1262
1263         tegra_first_level_clk_gate();
1264
1265         return tegra_ahci_power_off_rails(tegra_hpriv->power_rails);
1266 }
1267
1268 static int tegra_ahci_controller_resume(struct platform_device *pdev)
1269 {
1270         struct ata_host *host = dev_get_drvdata(&pdev->dev);
1271         struct tegra_ahci_host_priv *tegra_hpriv;
1272         unsigned long flags;
1273         int err;
1274
1275         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1276
1277         err = tegra_ahci_power_on_rails(tegra_hpriv->power_rails);
1278         if (err) {
1279                 pr_err("%s: fails to power on rails (%d)\n", __func__, err);
1280                 return err;
1281         }
1282
1283         err = tegra_first_level_clk_ungate();
1284         if (err < 0) {
1285                 pr_err("%s: flcg ungate failed\n", __func__);
1286                 return err;
1287         }
1288
1289         spin_lock_irqsave(&host->lock, flags);
1290         if (tegra_hpriv->pg_state == SATA_ON) {
1291                 dev_dbg(host->dev, "resume: SATA already powered on\n");
1292         } else {
1293                 dev_dbg(host->dev, "resume: powering on SATA...\n");
1294                 tegra_ahci_power_un_gate(host);
1295                 tegra_hpriv->pg_state = SATA_ON;
1296         }
1297         spin_unlock_irqrestore(&host->lock, flags);
1298         tegra_first_level_clk_gate();
1299
1300         return 0;
1301 }
1302
1303 static int tegra_ahci_suspend(struct platform_device *pdev, pm_message_t mesg)
1304 {
1305         struct ata_host *host = dev_get_drvdata(&pdev->dev);
1306         void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1307         u32 ctl;
1308         int rc;
1309
1310         dev_dbg(host->dev, "** entering %s: **\n", __func__);
1311         if (mesg.event & PM_EVENT_SLEEP) {
1312                 /*
1313                  * AHCI spec rev1.1 section 8.3.3:
1314                  * Software must disable interrupts prior to requesting a
1315                  * transition of the HBA to D3 state.
1316                  */
1317                 ctl = readl(mmio + HOST_CTL);
1318                 ctl &= ~HOST_IRQ_EN;
1319                 writel(ctl, mmio + HOST_CTL);
1320                 readl(mmio + HOST_CTL); /* flush */
1321         }
1322
1323         rc = ata_host_suspend(host, mesg);
1324         if (rc)
1325                 return rc;
1326
1327         return tegra_ahci_controller_suspend(pdev);
1328 }
1329
1330 static int tegra_ahci_resume(struct platform_device *pdev)
1331 {
1332         struct ata_host *host = dev_get_drvdata(&pdev->dev);
1333         int rc;
1334         u32 val;
1335
1336         dev_dbg(host->dev, "** entering %s: **\n", __func__);
1337         rc = tegra_ahci_controller_resume(pdev);
1338         if (rc != 0)
1339                 return rc;
1340
1341         rc = tegra_ahci_controller_init(g_tegra_hpriv, 1);
1342         if (rc != 0) {
1343                 dev_err(host->dev, "TEGRA SATA init failed in resume\n");
1344                 return rc;
1345         }
1346
1347         if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
1348                 rc = ahci_reset_controller(host);
1349                 if (rc)
1350                         return rc;
1351
1352                 val = misc_readl(SATA_AUX_RX_STAT_INT_0);
1353                 if (val && SATA_RX_STAT_INT_DISABLE) {
1354                         val &= ~SATA_RX_STAT_INT_DISABLE;
1355                         misc_writel(val, SATA_AUX_RX_STAT_INT_0);
1356                 }
1357
1358                 ahci_init_controller(host);
1359         }
1360
1361         ata_host_resume(host);
1362
1363         return 0;
1364 }
1365
1366 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
1367 static int tegra_ahci_runtime_suspend(struct device *dev)
1368 {
1369         struct ata_host *host;
1370         struct tegra_ahci_host_priv *tegra_hpriv;
1371         bool pg_ok;
1372         unsigned long flags;
1373         int err = 0;
1374
1375         host = dev_get_drvdata(dev);
1376         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1377
1378         spin_lock_irqsave(&host->lock, flags);
1379
1380         switch (tegra_hpriv->pg_state) {
1381         case SATA_OFF:
1382                 dev_dbg(dev, "** rt-suspend: already power gated **\n");
1383                 break;
1384
1385         case SATA_ABORT_OFF:
1386                 dev_dbg(dev, "** rt-suspend: abort suspend **\n");
1387                 tegra_hpriv->pg_state = SATA_ON;
1388                 tegra_ahci_dequeue_qcs(tegra_hpriv);
1389                 err = -EBUSY;
1390                 break;
1391
1392         case SATA_ON:
1393         case SATA_GOING_OFF:
1394                 if (tegra_ahci_are_all_ports_idle(host)) {
1395                         /* if all ports are in idle, do power-gate */
1396                         dev_dbg(dev, "** rt-suspend: power-down sata (%u) **\n",
1397                                         tegra_hpriv->pg_state);
1398 #ifdef TEGRA_AHCI_CONTEXT_RESTORE
1399                         pg_ok = tegra_ahci_power_gate(host);
1400 #else
1401                         pg_ok = tegra_ahci_pad_suspend(host);
1402 #endif
1403                         dev_dbg(dev, "** rt-suspend: done **\n");
1404                         if (pg_ok) {
1405                                 tegra_hpriv->pg_state = SATA_OFF;
1406                         } else {
1407                                 dev_err(dev, "** rt-suspend: abort pg **\n");
1408 #ifdef TEGRA_AHCI_CONTEXT_RESTORE
1409                                 tegra_ahci_abort_power_gate(host);
1410 #else
1411                                 tegra_ahci_abort_pad_suspend(host);
1412 #endif
1413                                 tegra_hpriv->pg_state = SATA_ON;
1414                                 err = -EBUSY;
1415                         }
1416                 } else {
1417                         dev_dbg(dev, "** rt-suspend: port not idle (%u) **\n",
1418                                         tegra_hpriv->pg_state);
1419                         err = -EBUSY;
1420                 }
1421                 break;
1422
1423         case SATA_GOING_ON:
1424         default:
1425                 dev_err(dev, "** rt-suspend: bad state (%u) **\n",
1426                         tegra_hpriv->pg_state);
1427                 WARN_ON(1);
1428                 err = -EBUSY;
1429                 break;
1430
1431         }
1432
1433         spin_unlock_irqrestore(&host->lock, flags);
1434
1435         return err;
1436 }
1437
1438 static int tegra_ahci_runtime_resume(struct device *dev)
1439 {
1440         struct ata_host *host;
1441         struct tegra_ahci_host_priv *tegra_hpriv;
1442         int err = 0;
1443
1444         host = dev_get_drvdata(dev);
1445         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1446
1447         if (tegra_hpriv->pg_state == SATA_ON) {
1448                 dev_dbg(dev, "** rt-resume: already power ungated **\n");
1449                 goto exit;
1450         }
1451
1452         if ((tegra_hpriv->pg_state == SATA_OFF) ||
1453             (tegra_hpriv->pg_state == SATA_GOING_ON)) {
1454                 dev_dbg(dev, "** rt-resume: power-up sata (%u) **\n",
1455                                 tegra_hpriv->pg_state);
1456 #ifdef TEGRA_AHCI_CONTEXT_RESTORE
1457                 tegra_ahci_power_un_gate(host);
1458 #else
1459                 tegra_ahci_pad_resume(host);
1460 #endif
1461                 dev_dbg(dev, "** rt-resume: done **\n");
1462                 tegra_hpriv->pg_state = SATA_ON;
1463
1464                 /* now qc_issue all qcs in the qc_list */
1465                 tegra_ahci_dequeue_qcs(tegra_hpriv);
1466
1467         } else {
1468                 dev_err(dev, "** rt-resume: bad state (%u) **\n",
1469                                 tegra_hpriv->pg_state);
1470                 WARN_ON(1);
1471                 err = -EBUSY;
1472         }
1473
1474
1475 exit:
1476
1477         return err;
1478 }
1479
1480 #endif
1481
1482 #ifdef TEGRA_AHCI_CONTEXT_RESTORE
1483 static u16 pg_save_bar5_registers[] = {
1484         0x018,  /* T_AHCI_HBA_CCC_PORTS */
1485         0x004,  /* T_AHCI_HBA_GHC */
1486         0x014,  /* T_AHCI_HBA_CCC_CTL - OP (optional) */
1487         0x01C,  /* T_AHCI_HBA_EM_LOC */
1488         0x020   /* T_AHCI_HBA_EM_CTL - OP */
1489 };
1490
1491 static u16 pg_save_bar5_port_registers[] = {
1492         0x100,  /* T_AHCI_PORT_PXCLB */
1493         0x104,  /* T_AHCI_PORT_PXCLBU */
1494         0x108,  /* T_AHCI_PORT_PXFB */
1495         0x10C,  /* T_AHCI_PORT_PXFBU */
1496         0x114,  /* T_AHCI_PORT_PXIE */
1497         0x118,  /* T_AHCI_PORT_PXCMD */
1498         0x12C,  /* T_AHCI_PORT_PXSCTL */
1499         0x144   /* T_AHCI_PORT_PXDEVSLP */
1500 };
1501
1502 /*
1503  * pg_save_bar5_bkdr_registers:
1504  *    These registers in BAR5 are read only.
1505  * To restore back those register values, write the saved value
1506  *    to the registers specified in pg_restore_bar5_bkdr_registers[].
1507  *    These pg_restore_bar5_bkdr_registers[] are in SATA_CONFIG space.
1508  */
1509 static u16 pg_save_bar5_bkdr_registers[] = {
1510         /* Save and restore via bkdr writes */
1511         0x000,  /* T_AHCI_HBA_CAP */
1512         0x00C,  /* T_AHCI_HBA_PI */
1513         0x024   /* T_AHCI_HBA_CAP2 */
1514 };
1515
1516 static u16 pg_restore_bar5_bkdr_registers[] = {
1517         /* Save and restore via bkdr writes */
1518         0x300,  /* BKDR of T_AHCI_HBA_CAP */
1519         0x33c,  /* BKDR of T_AHCI_HBA_PI */
1520         0x330   /* BKDR of T_AHCI_HBA_CAP2 */
1521 };
1522
1523 /* These registers are saved for each port */
1524 static u16 pg_save_bar5_bkdr_port_registers[] = {
1525         0x120,  /* NV_PROJ__SATA0_CHX_AHCI_PORT_PXTFD  */
1526         0x124,  /* NV_PROJ__SATA0_CHX_AHCI_PORT_PXSIG */
1527         0x128   /* NV_PROJ__SATA0_CHX_AHCI_PORT_PXSSTS */
1528 };
1529
1530 static u16 pg_restore_bar5_bkdr_port_registers[] = {
1531         /* Save and restore via bkdr writes */
1532         0x790,  /* BKDR of NV_PROJ__SATA0_CHX_AHCI_PORT_PXTFD  */
1533         0x794,  /* BKDR of NV_PROJ__SATA0_CHX_AHCI_PORT_PXSIG */
1534         0x798   /* BKDR of NV_PROJ__SATA0_CHX_AHCI_PORT_PXSSTS */
1535 };
1536
1537 static u16 pg_save_config_registers[] = {
1538         0x004,  /* T_SATA0_CFG_1 */
1539         0x00C,  /* T_SATA0_CFG_3 */
1540         0x024,  /* T_SATA0_CFG_9 */
1541         0x028,  /* T_SATA0_CFG_10 */
1542         0x030,  /* T_SATA0_CFG_12 */
1543         0x034,  /* T_SATA0_CFG_13 */
1544         0x038,  /* T_SATA0_CFG_14 */
1545         0x03C,  /* T_SATA0_CFG_15 */
1546         0x040,  /* T_SATA0_CFG_16 */
1547         0x044,  /* T_SATA0_CFG_17 */
1548         0x048,  /* T_SATA0_CFG_18 */
1549         0x0B0,  /* T_SATA0_MSI_CTRL */
1550         0x0B4,  /* T_SATA0_MSI_ADDR1 */
1551         0x0B8,  /* T_SATA0_MSI_ADDR2 */
1552         0x0BC,  /* T_SATA0_MSI_DATA */
1553         0x0C0,  /* T_SATA0_MSI_QUEUE */
1554         0x0EC,  /* T_SATA0_MSI_MAP */
1555         0x124,  /* T_SATA0_CFG_PHY_POWER */
1556         0x128,  /* T_SATA0_CFG_PHY_POWER_1 */
1557         0x12C,  /* T_SATA0_CFG_PHY_1 */
1558         0x174,  /* T_SATA0_CFG_LINK_0 */
1559         0x178,  /* T_SATA0_CFG_LINK_1 */
1560         0x1D0,  /* MCP_SATA0_CFG_TRANS_0 */
1561         0x238,  /* T_SATA0_ALPM_CTRL */
1562         0x30C,  /* T_SATA0_AHCI_HBA_CYA_0 */
1563         0x320,  /* T_SATA0_AHCI_HBA_SPARE_1 */
1564         0x324,  /* T_SATA0_AHCI_HBA_SPARE_2 */
1565         0x328,  /* T_SATA0_AHCI_HBA_DYN_CLK_CLAMP */
1566         0x32C,  /* T_SATA0_AHCI_CFG_ERR_CTRL */
1567         0x338,  /* T_SATA0_AHCI_HBA_CYA_1 */
1568         0x340,  /* T_SATA0_AHCI_HBA_PRE_STAGING_CONTROL */
1569         0x430,  /* T_SATA0_CFG_FPCI_0 */
1570         0x494,  /* T_SATA0_CFG_ESATA_CTRL */
1571         0x4A0,  /* T_SATA0_CYA1 */
1572         0x4B0,  /* T_SATA0_CFG_GLUE */
1573         0x534,  /* T_SATA0_PHY_CTRL */
1574         0x540,  /* T_SATA0_CTRL */
1575         0x550,  /* T_SATA0_DBG0 */
1576         0x554   /* T_SATA0_LOW_POWER_COUNT */
1577 };
1578
1579 static u16 pg_save_config_port_registers[] = {
1580         /* Save and restore per port */
1581         /* need to have port selected */
1582         0x530,  /* T_SATA0_CHXCFG1 */
1583         0x684,  /* T_SATA0_CHX_MISC */
1584         0x700,  /* T_SATA0_CHXCFG3 */
1585         0x704,  /* T_SATA0_CHXCFG4_CHX */
1586         0x690,  /* T_SATA0_CHX_PHY_CTRL1_GEN1 */
1587         0x694,  /* T_SATA0_CHX_PHY_CTRL1_GEN2 */
1588         0x698,  /* T_SATA0_CHX_PHY_CTRL1_GEN3 */
1589         0x69C,  /* T_SATA0_CHX_PHY_CTRL_2 */
1590         0x6B0,  /* T_SATA0_CHX_PHY_CTRL_3 */
1591         0x6B4,  /* T_SATA0_CHX_PHY_CTRL_4 */
1592         0x6B8,  /* T_SATA0_CHX_PHY_CTRL_5 */
1593         0x6BC,  /* T_SATA0_CHX_PHY_CTRL_6 */
1594         0x714,  /* T_SATA0_PRBS_CHX - OP */
1595         0x750,  /* T_SATA0_CHX_LINK0 */
1596         0x7F0   /* T_SATA0_CHX_GLUE */
1597 };
1598
1599 static u16 pg_save_ipfs_registers[] = {
1600         0x094,  /* SATA_FPCI_BAR5_0 */
1601         0x0C0,  /* SATA_MSI_BAR_SZ_0 */
1602         0x0C4,  /* SATA_MSI_AXI_BAR_ST_0 */
1603         0x0C8,  /* SATA_MSI_FPCI_BAR_ST_0 */
1604         0x140,  /* SATA_MSI_EN_VEC0_0 */
1605         0x144,  /* SATA_MSI_EN_VEC1_0 */
1606         0x148,  /* SATA_MSI_EN_VEC2_0 */
1607         0x14C,  /* SATA_MSI_EN_VEC3_0 */
1608         0x150,  /* SATA_MSI_EN_VEC4_0 */
1609         0x154,  /* SATA_MSI_EN_VEC5_0 */
1610         0x158,  /* SATA_MSI_EN_VEC6_0 */
1611         0x15C,  /* SATA_MSI_EN_VEC7_0 */
1612         0x180,  /* SATA_CONFIGURATION_0 */
1613         0x184,  /* SATA_FPCI_ERROR_MASKS_0 */
1614         0x188,  /* SATA_INTR_MASK_0 */
1615         0x1A0,  /* SATA_CFG_REVID_0 */
1616         0x198,  /* SATA_IPFS_INTR_ENABLE_0 */
1617         0x1BC,  /* SATA_CLKGATE_HYSTERSIS_0 */
1618         0x1DC   /* SATA_SATA_MCCIF_FIFOCTRL_0 */
1619 };
1620
1621 static void tegra_ahci_save_regs(u32 **save_addr,
1622                                  u32 reg_base,
1623                                  u16 reg_array[],
1624                                  u32 regs)
1625 {
1626         u32 i;
1627         u32 *dest = *save_addr;
1628         void __iomem *base = IO_ADDRESS(reg_base);
1629
1630         for (i = 0; i < regs; ++i, ++dest) {
1631                 *dest = readl(base + (u32)reg_array[i]);
1632                 AHCI_DBG_PRINT("save: [0x%x]=0x%08x\n",
1633                                (reg_base+(u32)reg_array[i]), *dest);
1634         }
1635         *save_addr = dest;
1636 }
1637
1638 static void tegra_ahci_restore_regs(void **save_addr,
1639                                     u32 reg_base,
1640                                     u16 reg_array[],
1641                                     u32 regs)
1642 {
1643         u32 i;
1644         u32 *src = *save_addr;
1645         void __iomem *base = IO_ADDRESS(reg_base);
1646
1647         for (i = 0; i < regs; ++i, ++src) {
1648                 writel(*src, base + (u32)reg_array[i]);
1649                 AHCI_DBG_PRINT("restore: [0x%x]=0x%08x\n",
1650                                 (reg_base+(u32)reg_array[i]), *src);
1651         }
1652         *save_addr = src;
1653 }
1654
1655 static void tegra_ahci_pg_save_registers(struct ata_host *host)
1656 {
1657         struct tegra_ahci_host_priv *tegra_hpriv;
1658         u32 *pg_save;
1659         u32 regs;
1660         int i;
1661
1662         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1663         pg_save = tegra_hpriv->pg_save;
1664
1665         /*
1666          * Driver should save/restore the registers in the order of
1667          * IPFS, CFG, Ext CFG, BAR5.
1668          */
1669
1670         /* save IPFS registers */
1671         regs = ARRAY_SIZE(pg_save_ipfs_registers);
1672         tegra_ahci_save_regs(&pg_save, TEGRA_SATA_BASE,
1673                              pg_save_ipfs_registers, regs);
1674         /* after the call, pg_save should point to the next address to save */
1675
1676         /* save CONFIG registers */
1677         regs = ARRAY_SIZE(pg_save_config_registers);
1678         tegra_ahci_save_regs(&pg_save, TEGRA_SATA_CONFIG_BASE,
1679                              pg_save_config_registers, regs);
1680
1681         /* save CONFIG per port registers */
1682         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i) {
1683                 scfg_writel((1 << i), T_SATA0_INDEX_OFFSET);
1684                 regs = ARRAY_SIZE(pg_save_config_port_registers);
1685                 tegra_ahci_save_regs(&pg_save, TEGRA_SATA_CONFIG_BASE,
1686                                      pg_save_config_port_registers, regs);
1687         }
1688         scfg_writel(SATA0_NONE_SELECTED, T_SATA0_INDEX_OFFSET);
1689
1690         /* save BAR5 registers */
1691         regs = ARRAY_SIZE(pg_save_bar5_registers);
1692         tegra_ahci_save_regs(&pg_save, TEGRA_SATA_BAR5_BASE,
1693                              pg_save_bar5_registers, regs);
1694
1695         /* save BAR5 port_registers */
1696         regs = ARRAY_SIZE(pg_save_bar5_port_registers);
1697         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i)
1698                 tegra_ahci_save_regs(&pg_save, TEGRA_SATA_BAR5_BASE + (0x80*i),
1699                                      pg_save_bar5_port_registers, regs);
1700
1701         /* save bkdr registers */
1702         regs = ARRAY_SIZE(pg_save_bar5_bkdr_registers);
1703         tegra_ahci_save_regs(&pg_save, TEGRA_SATA_BAR5_BASE,
1704                              pg_save_bar5_bkdr_registers, regs);
1705
1706         /* and save bkdr per_port registers */
1707         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i) {
1708                 scfg_writel((1 << i), T_SATA0_INDEX_OFFSET);
1709                 regs = ARRAY_SIZE(pg_save_bar5_bkdr_port_registers);
1710                 tegra_ahci_save_regs(&pg_save, TEGRA_SATA_BAR5_BASE + (0x80*i),
1711                                      pg_save_bar5_bkdr_port_registers,
1712                                      regs);
1713         }
1714         scfg_writel(SATA0_NONE_SELECTED, T_SATA0_INDEX_OFFSET);
1715 }
1716
1717 static void tegra_ahci_pg_restore_registers(struct ata_host *host)
1718 {
1719         struct tegra_ahci_host_priv *tegra_hpriv;
1720         void *pg_save;
1721         u32 regs, val;
1722         int i;
1723
1724         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1725         pg_save = tegra_hpriv->pg_save;
1726
1727         /*
1728          * Driver should restore the registers in the order of
1729          * IPFS, CFG, Ext CFG, BAR5.
1730          */
1731
1732         /* restore IPFS registers */
1733         regs = ARRAY_SIZE(pg_save_ipfs_registers);
1734         tegra_ahci_restore_regs(&pg_save, TEGRA_SATA_BASE,
1735                                 pg_save_ipfs_registers, regs);
1736         /* after the call, pg_save should point to the next addr to restore */
1737
1738         /* restore CONFIG registers */
1739         regs = ARRAY_SIZE(pg_save_config_registers);
1740         tegra_ahci_restore_regs(&pg_save, TEGRA_SATA_CONFIG_BASE,
1741                                 pg_save_config_registers, regs);
1742
1743         /* restore CONFIG per port registers */
1744         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i) {
1745                 scfg_writel((1 << i), T_SATA0_INDEX_OFFSET);
1746                 regs = ARRAY_SIZE(pg_save_config_port_registers);
1747                 tegra_ahci_restore_regs(&pg_save, TEGRA_SATA_CONFIG_BASE,
1748                                         pg_save_config_port_registers,
1749                                         regs);
1750         }
1751         scfg_writel(SATA0_NONE_SELECTED, T_SATA0_INDEX_OFFSET);
1752
1753         /* restore BAR5 registers */
1754         regs = ARRAY_SIZE(pg_save_bar5_registers);
1755         tegra_ahci_restore_regs(&pg_save, TEGRA_SATA_BAR5_BASE,
1756                                 pg_save_bar5_registers, regs);
1757
1758         /* restore BAR5 port_registers */
1759         regs = ARRAY_SIZE(pg_save_bar5_port_registers);
1760         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i)
1761                 tegra_ahci_restore_regs(&pg_save, TEGRA_SATA_BAR5_BASE+(0x80*i),
1762                                         pg_save_bar5_port_registers, regs);
1763
1764         /* restore bkdr registers */
1765         regs = ARRAY_SIZE(pg_restore_bar5_bkdr_registers);
1766         tegra_ahci_restore_regs(&pg_save, TEGRA_SATA_CONFIG_BASE,
1767                              pg_restore_bar5_bkdr_registers, regs);
1768
1769         /* and restore BAR5 bkdr per_port registers */
1770         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i) {
1771                 scfg_writel((1 << i), T_SATA0_INDEX_OFFSET);
1772                 regs = ARRAY_SIZE(pg_restore_bar5_bkdr_port_registers);
1773                 tegra_ahci_restore_regs(&pg_save, TEGRA_SATA_CONFIG_BASE,
1774                                         pg_restore_bar5_bkdr_port_registers,
1775                                         regs);
1776         }
1777         scfg_writel(SATA0_NONE_SELECTED, T_SATA0_INDEX_OFFSET);
1778
1779         /* program class code and programming interface for AHCI */
1780         val = scfg_readl(TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE);
1781         val |= TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE_EN;
1782         scfg_writel(val, TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE);
1783         scfg_writel(TEGRA_PRIVATE_AHCI_CC_BKDR_PGM, TEGRA_PRIVATE_AHCI_CC_BKDR);
1784         val &= ~TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE_EN;
1785         scfg_writel(val, TEGRA_PRIVATE_AHCI_CC_BKDR_OVERRIDE);
1786 }
1787 #endif
1788 static u32 tegra_ahci_port_error(struct ata_port *ap)
1789 {
1790         void __iomem *port_mmio = ahci_port_base(ap);
1791         u32 err_status;
1792
1793         err_status = readl(port_mmio + PORT_IRQ_STAT);
1794         /* excludes PhyRdy and Connect Change status */
1795         err_status &= (PORT_IRQ_ERROR & (~(PORT_IRQ_PHYRDY|PORT_IRQ_CONNECT)));
1796         return err_status;
1797 }
1798
1799 static bool tegra_ahci_check_errors(struct ata_host *host)
1800 {       int i;
1801         struct ata_port *ap;
1802         u32 err;
1803
1804         for (i = 0; i < host->n_ports; i++) {
1805                 ap = host->ports[i];
1806                 err = tegra_ahci_port_error(ap);
1807                 if (err) {
1808                         dev_err(host->dev,
1809                                 "pg-chk-err = 0x%08x on port %d\n", err, i);
1810                         return true;
1811                 }
1812         }
1813         return false;
1814 }
1815
1816 void tegra_ahci_iddqlane_config(void)
1817 {
1818         u32 val;
1819         u32 dat;
1820
1821         /* wait for SATA_PADPLL_IDDQ2LANE_SLUMBER_DLY = 3 microseconds. */
1822         val = clk_readl(CLK_RST_SATA_PLL_CFG1_REG);
1823         val &= ~IDDQ2LANE_SLUMBER_DLY_MASK;
1824         val |= IDDQ2LANE_SLUMBER_DLY_3MS;
1825         clk_writel(val, CLK_RST_SATA_PLL_CFG1_REG);
1826
1827         /* wait for delay of IDDQ2LAND_SLUMBER_DLY */
1828         val = clk_readl(CLK_RST_SATA_PLL_CFG1_REG);
1829         dat = (val & IDDQ2LANE_SLUMBER_DLY_MASK) >> IDDQ2LANE_SLUMBER_DLY_SHIFT;
1830         udelay(dat);
1831
1832 }
1833 void tegra_ahci_put_sata_in_iddq()
1834 {
1835         u32 val;
1836         u32 dat;
1837
1838         /*
1839          * Hw wake up is not needed:
1840          * Driver/RM shall place the SATA PHY and SATA PADPLL in IDDQ.
1841          * SATA_PADPLL_RESET_SWCTL =1
1842          * SATA_PADPLL_RESET_OVERRIDE_VALUE=1
1843          * SATA_PADPHY_IDDQ_SWCTL=1
1844          * SATA_PADPHY_IDDQ_OVERRIDE_VALUE=1
1845          */
1846
1847         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
1848         val &= ~(PADPLL_RESET_SWCTL_MASK | PADPLL_RESET_OVERRIDE_VALUE_MASK);
1849         val |= (PADPLL_RESET_SWCTL_ON | PADPLL_RESET_OVERRIDE_VALUE_ON);
1850         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
1851
1852         /* Wait for time specified in SATA_LANE_IDDQ2_PADPLL_IDDQ */
1853         val = clk_readl(CLK_RST_SATA_PLL_CFG1_REG);
1854         dat = (val & IDDQ2LANE_IDDQ_DLY_MASK) >> IDDQ2LANE_IDDQ_DLY_SHIFT;
1855         udelay(dat);
1856
1857         val = xusb_readl(XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
1858         val |= (PLL_PWR_OVRD_MASK | PLL_IDDQ_MASK | PLL_RST_MASK);
1859         xusb_writel(val, XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
1860
1861 }
1862 void tegra_ahci_clr_clk_rst_cnt_rst_dev(void)
1863 {
1864         u32 val;
1865
1866         val = clk_readl(CLK_RST_CONTROLLER_RST_DEV_V_CLR_0);
1867         val |= (SWR_SATA_OOB_RST | SWR_SATA_RST);
1868         clk_writel(val, CLK_RST_CONTROLLER_RST_DEV_V_CLR_0);
1869
1870         val = clk_readl(CLK_RST_CONTROLLER_RST_DEV_W_CLR_0);
1871         val |= SWR_SATACOLD_RST;
1872         clk_writel(val, CLK_RST_CONTROLLER_RST_DEV_W_CLR_0);
1873
1874 }
1875 void tegra_ahci_set_clk_rst_cnt_rst_dev(void)
1876 {
1877
1878         u32 val;
1879
1880         val = clk_readl(CLK_RST_CONTROLLER_RST_DEV_V_CLR_0);
1881         val &= ~(SWR_SATA_OOB_RST | SWR_SATA_RST);
1882         clk_writel(val, CLK_RST_CONTROLLER_RST_DEV_V_CLR_0);
1883
1884         val = clk_readl(CLK_RST_CONTROLLER_RST_DEV_W_CLR_0);
1885         val &= ~SWR_SATACOLD_RST;
1886         clk_writel(val, CLK_RST_CONTROLLER_RST_DEV_W_CLR_0);
1887
1888 }
1889
1890 static void tegra_ahci_pad_config(void)
1891 {
1892         u32 val;
1893
1894         /* clear SW control of SATA PADPLL, SATA PHY and PLLE */
1895
1896         /* for SATA PHY IDDQ */
1897         val = xusb_readl(XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0);
1898         val &= ~(IDDQ_OVRD_MASK | IDDQ_MASK);
1899         xusb_writel(val, XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0);
1900
1901         /* for SATA PADPLL IDDQ */
1902         val = xusb_readl(XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
1903         val &= ~(PLL_PWR_OVRD_MASK | PLL_IDDQ_MASK | PLL_RST_MASK);
1904         xusb_writel(val, XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
1905
1906         /* PLLE related stuff*/
1907
1908         val = clk_readl(CLK_RST_CONTROLLER_PLLE_MISC_0);
1909         val &= ~(T124_PLLE_IDDQ_SWCTL_MASK | PLLE_IDDQ_OVERRIDE_VALUE_MASK);
1910         clk_writel(val, CLK_RST_CONTROLLER_PLLE_MISC_0);
1911
1912         clk_writel(CLK_RST_CONTROLLER_PLLE_MISC_0_VALUE,
1913                         CLK_RST_CONTROLLER_PLLE_MISC_0);
1914
1915         val = clk_readl(CLK_RST_CONTROLLER_PLLE_BASE_0);
1916         val |= PLLE_ENABLE;
1917         clk_writel(val, CLK_RST_CONTROLLER_PLLE_BASE_0);
1918
1919 }
1920
1921 static void tegra_ahci_abort_power_gate(struct ata_host *host)
1922 {
1923         u32 val;
1924         val = pmc_readl(APB_PMC_SATA_PWRGT_0_REG);
1925         val &= ~PG_INFO_MASK;
1926         pmc_writel(val, APB_PMC_SATA_PWRGT_0_REG);
1927 }
1928
1929 static bool tegra_ahci_power_gate(struct ata_host *host)
1930 {
1931         u32 val;
1932         u32 dat;
1933         struct tegra_ahci_host_priv *tegra_hpriv;
1934         int status;
1935
1936         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
1937
1938         val = pmc_readl(APB_PMC_SATA_PWRGT_0_REG);
1939         val &= ~PG_INFO_MASK;
1940         val |= PG_INFO_ON;
1941         pmc_writel(val, APB_PMC_SATA_PWRGT_0_REG);
1942
1943 #ifdef TEGRA_AHCI_CONTEXT_RESTORE
1944         tegra_ahci_pg_save_registers(host);
1945 #endif
1946         val = xusb_readl(XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0);
1947         val |= XUSB_PADCTL_IOPHY_MISC_IDDQ |
1948                 XUSB_PADCTL_IOPHY_MISC_IDDQ_OVRD;
1949         xusb_writel(val, XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0);
1950
1951         /*
1952          * Read SATA_AUX_MISC_CNTL_1_0 register L0_RX_IDLE_T_SAX field and
1953          * write that value into same register L0_RX_IDLE_T_NPG field.
1954          * And write 1 to L0_RX_IDLE_T_MUX field.
1955          */
1956         val = misc_readl(SATA_AUX_MISC_CNTL_1_REG);
1957         dat = val;
1958         dat &= L0_RX_IDLE_T_SAX_MASK;
1959         dat >>= L0_RX_IDLE_T_SAX_SHIFT;
1960         dat <<= L0_RX_IDLE_T_NPG_SHIFT;
1961         val &= ~L0_RX_IDLE_T_NPG_MASK;
1962         val |= dat;
1963         val |= L0_RX_IDLE_T_MUX_FROM_APB_MISC;
1964         val |= DEVSLP_OVERRIDE;
1965         misc_writel(val, SATA_AUX_MISC_CNTL_1_REG);
1966
1967         tegra_ahci_set_clk_rst_cnt_rst_dev();
1968
1969         /* abort PG if there are errors occurred */
1970         if (tegra_ahci_check_errors(host)) {
1971                 dev_err(host->dev, "** pg: errors; abort power gating **\n");
1972                 return false;
1973         }
1974         /* make sure all ports have no outstanding commands and are idle. */
1975         if (!tegra_ahci_are_all_ports_idle(host)) {
1976                 dev_err(host->dev, "** pg: cmds; abort power gating **\n");
1977                 return false;
1978         }
1979         tegra_ahci_put_sata_in_iddq();
1980
1981         val = pmc_readl(APBDEV_PMC_PWRGATE_TOGGLE_0);
1982         val |= PARTID_VALUE;
1983         val |= START;
1984         pmc_writel(val, APBDEV_PMC_PWRGATE_TOGGLE_0);
1985
1986         val = pmc_readl(APBDEV_PMC_PWRGATE_STATUS_0);
1987         val &= ~SAX_MASK;
1988         pmc_writel(val, APBDEV_PMC_PWRGATE_STATUS_0);
1989
1990         /* power off the sata */
1991         status = tegra_powergate_partition_with_clk_off(TEGRA_POWERGATE_SATA);
1992         if (status) {
1993                 dev_err(host->dev, "** failed to turn-off SATA (0x%x) **\n",
1994                                    status);
1995                 return false;
1996         }
1997
1998         return true;
1999 }
2000
2001 static bool tegra_ahci_power_un_gate(struct ata_host *host)
2002 {
2003         u32 val;
2004         u32 timeout;
2005         struct tegra_ahci_host_priv *tegra_hpriv;
2006         int status;
2007
2008         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
2009
2010         tegra_ahci_iddqlane_config();
2011
2012         val = pmc_readl(APBDEV_PMC_PWRGATE_TOGGLE_0);
2013         val |= PARTID_VALUE;
2014         val |= START;
2015         pmc_writel(val, APBDEV_PMC_PWRGATE_TOGGLE_0);
2016
2017         val = pmc_readl(APBDEV_PMC_PWRGATE_STATUS_0);
2018         val |= SAX_MASK;
2019         pmc_writel(val, APBDEV_PMC_PWRGATE_STATUS_0);
2020
2021         status = tegra_unpowergate_partition_with_clk_on(TEGRA_POWERGATE_SATA);
2022         if (status) {
2023                 dev_err(host->dev, "** failed to turn-on SATA (0x%x) **\n",
2024                                    status);
2025                 return false;
2026         }
2027
2028         /* deasset PADPLL and wait until it locks. */
2029         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
2030         val &= ~PADPLL_RESET_OVERRIDE_VALUE_MASK;
2031         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
2032
2033         tegra_ahci_clr_clk_rst_cnt_rst_dev();
2034
2035 #ifdef TEGRA_AHCI_CONTEXT_RESTORE
2036         /* restore registers */
2037         tegra_ahci_pg_restore_registers(host);
2038 #endif
2039         tegra_ahci_pad_config();
2040
2041         /*
2042          * Wait for SATA_AUX_PAD_PLL_CNTL_1_0_LOCKDET to turn 1 with a timeout
2043          * of 15 us.
2044          */
2045         timeout = 15;
2046         while (timeout--) {
2047                 udelay(1);
2048                 val = xusb_readl(XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
2049                 if (val & LOCKDET_FIELD)
2050                         break;
2051         }
2052         if (timeout == 0)
2053                 pr_err("%s: SATA_PAD_PLL is not locked in 15us.\n", __func__);
2054
2055         /*
2056          * During the restoration of the registers, the driver would now need to
2057          * restore the register T_SATA0_CFG_POWER_GATE_SSTS_RESTORED after the
2058          * ssts_det, ssts_spd are restored. This register is used to tell the
2059          * controller whether a drive existed earlier or not and move the PHY
2060          * state machines into either HR_slumber or not.
2061          */
2062         val = scfg_readl(T_SATA0_CFG_POWER_GATE);
2063         val &= ~POWER_GATE_SSTS_RESTORED_MASK;
2064         val |= POWER_GATE_SSTS_RESTORED_YES;
2065         scfg_writel(val, T_SATA0_CFG_POWER_GATE);
2066
2067
2068         /*
2069          * Driver needs to switch the rx_idle_t driven source back to from
2070          * Sata controller after SAX is power-ungated.
2071          */
2072         val = misc_readl(SATA_AUX_MISC_CNTL_1_REG);
2073         val &= ~DEVSLP_OVERRIDE;
2074         val &= ~L0_RX_IDLE_T_MUX_MASK;
2075         val |= L0_RX_IDLE_T_MUX_FROM_SATA;
2076         misc_writel(val, SATA_AUX_MISC_CNTL_1_REG);
2077
2078         /* Set the bits in the CAR to allow HW based low power sequencing. */
2079         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
2080         val &= ~PADPLL_RESET_SWCTL_MASK;
2081         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
2082
2083         /*
2084          * power un-gating process is complete by clearing
2085          * APBDEV_PMC_SATA_PWRGT_0.Pmc2sata_pg_info = 0
2086          */
2087         val = pmc_readl(APB_PMC_SATA_PWRGT_0_REG);
2088         val &= ~PG_INFO_MASK;
2089         pmc_writel(val, APB_PMC_SATA_PWRGT_0_REG);
2090
2091         return true;
2092 }
2093
2094 static enum port_idle_status tegra_ahci_is_port_idle(struct ata_port *ap)
2095 {
2096         void __iomem *port_mmio = ahci_port_base(ap);
2097
2098         if (readl(port_mmio + PORT_CMD_ISSUE) ||
2099             readl(port_mmio + PORT_SCR_ACT))
2100                 return PORT_IS_NOT_IDLE;
2101         return PORT_IS_IDLE;
2102 }
2103
2104 /* check if all supported ports are idle (no outstanding commands) */
2105 static bool tegra_ahci_are_all_ports_idle(struct ata_host *host)
2106 {       int i;
2107         struct ata_port *ap;
2108
2109         for (i = 0; i < host->n_ports; i++) {
2110                 ap = host->ports[i];
2111                 if (ap && (tegra_ahci_is_port_idle(ap) == PORT_IS_NOT_IDLE))
2112                         return false;
2113         }
2114         return true;
2115 }
2116
2117 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
2118 static void tegra_ahci_abort_pad_suspend(struct ata_host *host)
2119 {
2120         /*No implementation*/
2121 }
2122
2123 static bool tegra_ahci_pad_suspend(struct ata_host *host)
2124 {
2125         u32 val;
2126         struct tegra_ahci_host_priv *tegra_hpriv;
2127
2128         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
2129
2130         /* Set the bits in the CAR to allow HW based low power sequencing. */
2131         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
2132         val |= PADPLL_RESET_SWCTL_MASK;
2133         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
2134
2135         val = xusb_readl(XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0);
2136         val |= XUSB_PADCTL_IOPHY_MISC_IDDQ |
2137                 XUSB_PADCTL_IOPHY_MISC_IDDQ_OVRD;
2138         xusb_writel(val, XUSB_PADCTL_IOPHY_MISC_PAD_S0_CTL_1_0);
2139
2140         /* abort PG if there are errors occurred */
2141         if (tegra_ahci_check_errors(host)) {
2142                 dev_err(host->dev, "** pg: errors; abort power gating **\n");
2143                 return false;
2144         }
2145         /* make sure all ports have no outstanding commands and are idle. */
2146         if (!tegra_ahci_are_all_ports_idle(host)) {
2147                 dev_err(host->dev, "** pg: cmds; abort power gating **\n");
2148                 return false;
2149         }
2150         tegra_ahci_put_sata_in_iddq();
2151
2152         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
2153         val |= (SATA_SEQ_PADPLL_PD_INPUT_VALUE |
2154                 SATA_SEQ_LANE_PD_INPUT_VALUE | SATA_SEQ_RESET_INPUT_VALUE);
2155         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
2156
2157         tegra_first_level_clk_gate();
2158
2159         return true;
2160 }
2161
2162 static bool tegra_ahci_pad_resume(struct ata_host *host)
2163 {
2164         u32 val;
2165         u32 timeout;
2166         struct tegra_ahci_host_priv *tegra_hpriv;
2167
2168         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
2169
2170         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
2171         val &= ~(SATA_SEQ_PADPLL_PD_INPUT_VALUE |
2172                 SATA_SEQ_LANE_PD_INPUT_VALUE | SATA_SEQ_RESET_INPUT_VALUE);
2173         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
2174
2175         if (tegra_first_level_clk_ungate() < 0) {
2176                 pr_err("%s: flcg ungate failed\n", __func__);
2177                 return false;
2178         }
2179
2180         tegra_ahci_iddqlane_config();
2181
2182         /* deasset PADPLL and wait until it locks. */
2183         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
2184         val &= ~PADPLL_RESET_OVERRIDE_VALUE_MASK;
2185         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
2186
2187         tegra_ahci_pad_config();
2188
2189         /*
2190          * Wait for SATA_AUX_PAD_PLL_CNTL_1_0_LOCKDET to turn 1 with a timeout
2191          * of 15 us.
2192          */
2193         timeout = 15;
2194         while (timeout--) {
2195                 udelay(1);
2196                 val = xusb_readl(XUSB_PADCTL_IOPHY_PLL_S0_CTL1_0);
2197                 if (val & LOCKDET_FIELD)
2198                         break;
2199         }
2200         if (timeout == 0)
2201                 pr_err("%s: SATA_PAD_PLL is not locked in 15us.\n", __func__);
2202
2203         /* Set the bits in the CAR to allow HW based low power sequencing. */
2204         val = clk_readl(CLK_RST_SATA_PLL_CFG0_REG);
2205         val &= ~PADPLL_RESET_SWCTL_MASK;
2206         clk_writel(val, CLK_RST_SATA_PLL_CFG0_REG);
2207
2208         return true;
2209 }
2210
2211 static void tegra_ahci_to_add_idle_timer(struct ata_host *host)
2212 {
2213         struct tegra_ahci_host_priv *tegra_hpriv;
2214
2215         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
2216
2217         /* note: the routine is called from interrupt context */
2218         spin_lock(&host->lock);
2219         /* start idle-timer if all ports have no outstanding commands */
2220         if (tegra_ahci_are_all_ports_idle(host)) {
2221                 /* adjust tegra_ahci_idle_time to minimum if it is too small */
2222                 tegra_ahci_idle_time = max((u32)TEGRA_AHCI_MIN_IDLE_TIME,
2223                                            tegra_ahci_idle_time);
2224                 tegra_hpriv->idle_timer.expires =
2225                         ata_deadline(jiffies, tegra_ahci_idle_time);
2226                 mod_timer(&tegra_hpriv->idle_timer,
2227                           tegra_hpriv->idle_timer.expires);
2228         }
2229         spin_unlock(&host->lock);
2230 }
2231
2232 static void tegra_ahci_idle_timer(unsigned long arg)
2233 {
2234         struct ata_host *host = (void *)arg;
2235         struct tegra_ahci_host_priv *tegra_hpriv;
2236         unsigned long flags;
2237
2238         tegra_hpriv = (struct tegra_ahci_host_priv *)host->private_data;
2239
2240         spin_lock_irqsave(&host->lock, flags);
2241         if (tegra_hpriv->pg_state == SATA_ON)
2242                 tegra_hpriv->pg_state = SATA_GOING_OFF;
2243         else {
2244                 dev_err(host->dev, "idle_timer: bad state (%u)\n",
2245                                 tegra_hpriv->pg_state);
2246                 WARN_ON(1);
2247                 spin_unlock_irqrestore(&host->lock, flags);
2248                 return;
2249         }
2250         spin_unlock_irqrestore(&host->lock, flags);
2251         tegra_ahci_runtime_suspend(tegra_hpriv->dev);
2252 }
2253
2254 static int tegra_ahci_queue_one_qc(struct tegra_ahci_host_priv *tegra_hpriv,
2255                                    struct ata_queued_cmd *qc)
2256 {
2257         struct tegra_qc_list *qc_list;
2258
2259         qc_list = kmalloc(sizeof(struct tegra_qc_list), GFP_ATOMIC);
2260         if (!qc_list) {
2261                 dev_err(tegra_hpriv->dev, "failed to alloc qc_list\n");
2262                 return AC_ERR_SYSTEM;
2263         }
2264         qc_list->qc = qc;
2265         list_add_tail(&(qc_list->list), &(tegra_hpriv->qc_list));
2266         return 0;
2267 }
2268
2269 static void tegra_ahci_dequeue_qcs(struct tegra_ahci_host_priv *tegra_hpriv)
2270 {
2271         struct list_head *list, *next;
2272         struct tegra_qc_list *qc_list;
2273         struct ata_queued_cmd *qc;
2274
2275         /* now qc_issue all qcs in the qc_list */
2276         list_for_each_safe(list, next, &tegra_hpriv->qc_list) {
2277                 qc_list = list_entry(list, struct tegra_qc_list, list);
2278                 qc = qc_list->qc;
2279                 ahci_ops.qc_issue(qc);
2280                 list_del(list);
2281                 kfree(qc_list);
2282         }
2283 }
2284
2285 static unsigned int tegra_ahci_qc_issue(struct ata_queued_cmd *qc)
2286 {
2287         struct ata_port *ap = qc->ap;
2288         struct ata_host *host = ap->host;
2289         struct tegra_ahci_host_priv *tegra_hpriv = host->private_data;
2290
2291         /* stop the idle timer */
2292         if (timer_pending(&tegra_hpriv->idle_timer))
2293                 del_timer_sync(&tegra_hpriv->idle_timer);
2294
2295         /* note: host->lock is locked */
2296         switch (tegra_hpriv->pg_state) {
2297         case SATA_ON:
2298                 /* normal case, issue the qc */
2299                 return ahci_ops.qc_issue(qc);
2300         case SATA_GOING_OFF:
2301         case SATA_ABORT_OFF:
2302                 /* SATA is going OFF, let's abort the suspend */
2303                 dev_dbg(host->dev, "** qc_issue: going OFF **\n");
2304                 tegra_hpriv->pg_state = SATA_ABORT_OFF;
2305                 return tegra_ahci_queue_one_qc(tegra_hpriv, qc);
2306         case SATA_OFF:
2307                 dev_dbg(host->dev, "** qc_issue: request power-up sata **\n");
2308                 queue_work(tegra_ahci_work_q, &tegra_ahci_work);
2309                 tegra_hpriv->pg_state = SATA_GOING_ON;
2310                 /* continue with the following code to queue the qc */
2311         case SATA_GOING_ON:
2312                 return tegra_ahci_queue_one_qc(tegra_hpriv, qc);
2313         default:
2314                 dev_err(host->dev, "** qc_issue: bad state (%u) **\n",
2315                                         tegra_hpriv->pg_state);
2316                 WARN_ON(1);
2317                 return AC_ERR_SYSTEM;
2318         }
2319 }
2320
2321 static int tegra_ahci_hardreset(struct ata_link *link, unsigned int *class,
2322                                 unsigned long deadline)
2323 {
2324         struct ata_port *ap = link->ap;
2325         struct ata_host *host = ap->host;
2326         struct tegra_ahci_host_priv *tegra_hpriv = host->private_data;
2327         unsigned long flags;
2328         int rc;
2329
2330         if (tegra_hpriv->pg_state == SATA_OFF) {
2331                 dev_dbg(host->dev, "** hreset: request power-up sata **\n");
2332                 spin_lock_irqsave(&host->lock, flags);
2333                 rc = tegra_ahci_runtime_resume(tegra_hpriv->dev);
2334                 spin_unlock_irqrestore(&host->lock, flags);
2335                 /* rc == 0 means the request has been run successfully */
2336                 if (rc) {
2337                         dev_err(host->dev, "** hreset: rt_get()=%d **\n", rc);
2338                         WARN_ON(1);
2339                         return AC_ERR_SYSTEM;
2340                 }
2341                 tegra_hpriv->pg_state = SATA_ON;
2342         }
2343
2344         return ahci_ops.hardreset(link, class, deadline);
2345 }
2346
2347 static irqreturn_t tegra_ahci_interrupt(int irq, void *dev_instance)
2348 {
2349         irqreturn_t irq_retval;
2350         u32 val;
2351
2352         val = misc_readl(SATA_AUX_RX_STAT_INT_0);
2353         if (!(val && SATA_RX_STAT_INT_DISABLE)) {
2354                 val |= SATA_RX_STAT_INT_DISABLE;
2355                 misc_writel(val, SATA_AUX_RX_STAT_INT_0);
2356         }
2357
2358         irq_retval = ahci_interrupt(irq, dev_instance);
2359         if (irq_retval == IRQ_NONE)
2360                 return IRQ_NONE;
2361
2362 #ifdef CONFIG_PM
2363         tegra_ahci_to_add_idle_timer((struct ata_host *)dev_instance);
2364 #endif
2365
2366         return irq_retval;
2367 }
2368 #endif
2369 #endif
2370
2371 static int tegra_ahci_remove_one(struct platform_device *pdev)
2372 {
2373         struct ata_host *host = dev_get_drvdata(&pdev->dev);
2374         struct ahci_host_priv *hpriv;
2375
2376         BUG_ON(host == NULL);
2377         BUG_ON(host->iomap[AHCI_PCI_BAR] == NULL);
2378         hpriv = host->private_data;
2379
2380         tegra_ahci_controller_remove(pdev);
2381
2382         devm_iounmap(&pdev->dev, host->iomap[AHCI_PCI_BAR]);
2383         ata_host_detach(host);
2384
2385 #ifdef TEGRA_AHCI_CONTEXT_RESTORE
2386         /* Free PG save/restore area */
2387         devm_kfree(&pdev->dev, ((struct tegra_ahci_host_priv *)hpriv)->pg_save);
2388
2389 #endif
2390 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
2391         cancel_work_sync(&tegra_ahci_work);
2392         if (tegra_ahci_work_q)
2393                 destroy_workqueue(tegra_ahci_work_q);
2394 #endif
2395
2396         devm_kfree(&pdev->dev, hpriv);
2397
2398         return 0;
2399 }
2400 void tegra_ahci_sata_clk_gate(void)
2401 {
2402         u32 val;
2403
2404         if (!tegra_platform_is_silicon())
2405                 return;
2406
2407         val = clk_readl(CLK_RST_CONTROLLER_RST_DEV_W_SET);
2408         if (val & SET_CEC_RESET)
2409                 clk_writel(0x108, CLK_RST_CONTROLLER_RST_DEV_V_SET);
2410         val = clk_readl(CLK_RST_CONTROLLER_RST_DEV_W_SET);
2411         while (val & SET_CEC_RESET)
2412                 val = clk_readl(CLK_RST_CONTROLLER_RST_DEV_W_SET);
2413 }
2414
2415 static int tegra_ahci_init_one(struct platform_device *pdev)
2416 {
2417         struct device_node *np = pdev->dev.of_node;
2418         struct ata_port_info pi = ahci_port_info;
2419         const struct ata_port_info *ppi[] = { &pi, NULL };
2420         struct device *dev = &pdev->dev;
2421         struct ahci_host_priv *hpriv = NULL;
2422         struct tegra_ahci_host_priv *tegra_hpriv;
2423         struct tegra_ahci_platform_data *ahci_pdata;
2424         struct ata_host *host = NULL;
2425         int n_ports, i, rc = 0;
2426         struct resource *res, *irq_res;
2427         void __iomem *mmio;
2428
2429 #if defined(TEGRA_AHCI_CONTEXT_RESTORE)
2430         u32 save_size;
2431 #endif
2432         irq_handler_t irq_handler = ahci_interrupt;
2433
2434         VPRINTK("ENTER\n");
2435
2436         WARN_ON((int)ATA_MAX_QUEUE > AHCI_MAX_CMDS);
2437
2438         ata_print_version_once(&pdev->dev, DRV_VERSION);
2439
2440         /* Simple resource validation */
2441         if (pdev->num_resources != 3) {
2442                 dev_err(dev, "invalid number of resources\n");
2443                 dev_err(dev, "not enough SATA resources\n");
2444                 return -EINVAL;
2445         }
2446
2447         /* acquire bar resources */
2448         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2449         if (res == NULL)
2450                 return -EINVAL;
2451
2452         /* acquire IRQ resource */
2453         irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
2454         if (irq_res == NULL)
2455                 return -EINVAL;
2456         if (irq_res->start <= 0)
2457                 return -EINVAL;
2458
2459         /* allocate sizeof tegra_ahci_host_priv, which contains extra fields */
2460         hpriv = devm_kzalloc(dev, sizeof(struct tegra_ahci_host_priv),
2461                              GFP_KERNEL);
2462         if (!hpriv) {
2463                 rc = -ENOMEM;
2464                 goto fail;
2465         }
2466
2467 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
2468         tegra_ahci_work_q = alloc_workqueue("tegra_ahci_work_q",
2469                                         WQ_HIGHPRI | WQ_UNBOUND, 16);
2470         if (!tegra_ahci_work_q) {
2471                 dev_err(dev, "TEGRA WORKQUEUE SATA init failed\n");
2472                 goto fail;
2473         }
2474 #endif
2475
2476         hpriv->flags |= (unsigned long)pi.private_data;
2477         tegra_hpriv = (struct tegra_ahci_host_priv *)hpriv;
2478         tegra_hpriv->dev = dev;
2479         if (np) {
2480                 pdev->dev.coherent_dma_mask = DMA_BIT_MASK(64);
2481                 pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask;
2482                 tegra_hpriv->pexp_gpio =
2483                         of_get_named_gpio(np, "nvidia,pexp-gpio", 0);
2484                 if (!of_property_read_bool(np, "nvidia,enable-sata-port")) {
2485                         dev_err(dev, "Not able to find enable-sata-port property\n");
2486                         tegra_ahci_sata_clk_gate();
2487                         goto fail;
2488                 }
2489                 if (of_property_read_u32(np, "nvidia,sata-connector-type",
2490                         &tegra_hpriv->sata_connector) < 0) {
2491                         tegra_hpriv->sata_connector = MINI_SATA;
2492                 }
2493
2494         } else {
2495                 ahci_pdata = tegra_hpriv->dev->platform_data;
2496                 tegra_hpriv->pexp_gpio = ahci_pdata->pexp_gpio;
2497         }
2498         g_tegra_hpriv = tegra_hpriv;
2499
2500         /* Call tegra init routine */
2501         rc = tegra_ahci_controller_init(tegra_hpriv, 0);
2502         if (rc != 0) {
2503                 dev_err(dev, "TEGRA SATA init failed\n");
2504                 goto fail;
2505         }
2506
2507         /*
2508          * We reserve a table of 6 BARs in tegra_hpriv to store BARs.
2509          * Save the mapped AHCI_PCI_BAR address to the table.
2510          */
2511         mmio = devm_ioremap(dev, res->start, (res->end-res->start+1));
2512         tegra_hpriv->bars_table[AHCI_PCI_BAR] = mmio;
2513         hpriv->mmio = mmio;
2514
2515         /* save initial config */
2516         tegra_ahci_save_initial_config(pdev, hpriv);
2517         dev_dbg(dev, "past save init config\n");
2518
2519         /* prepare host */
2520         if (hpriv->cap & HOST_CAP_NCQ) {
2521                 pi.flags |= ATA_FLAG_NCQ;
2522                 pi.flags |= ATA_FLAG_FPDMA_AA;
2523         }
2524         if (hpriv->cap & HOST_CAP_PMP)
2525                 pi.flags |= ATA_FLAG_PMP;
2526
2527         /*
2528          * CAP.NP sometimes indicate the index of the last enabled
2529          * port, at other times, that of the last possible port, so
2530          * determining the maximum port number requires looking at
2531          * both CAP.NP and port_map.
2532          */
2533         n_ports = max(ahci_nr_ports(hpriv->cap), fls(hpriv->port_map));
2534         host = ata_host_alloc_pinfo(dev, ppi, n_ports);
2535         if (!host) {
2536                 rc = -ENOMEM;
2537                 goto fail;
2538         }
2539         host->private_data = hpriv;
2540         tegra_hpriv->host = host;
2541         host->iomap = tegra_hpriv->bars_table;
2542
2543         if (!(hpriv->cap & HOST_CAP_SSS))
2544                 host->flags |= ATA_HOST_PARALLEL_SCAN;
2545         else
2546                 pr_info("ahci: SSS flag set, parallel bus scan disabled\n");
2547
2548         for (i = 0; i < host->n_ports; i++) {
2549                 struct ata_port *ap = host->ports[i];
2550
2551                 /* set initial link pm policy */
2552                 ap->target_lpm_policy = ATA_LPM_UNKNOWN;
2553
2554                 /* disabled/not-implemented port */
2555                 if (!(hpriv->port_map & (1 << i)))
2556                         ap->ops = &ata_dummy_port_ops;
2557                 else
2558                         ap->target_lpm_policy = ATA_LPM_MIN_POWER;
2559         }
2560
2561         rc = ahci_reset_controller(host);
2562         if (rc) {
2563                 dev_err(dev, "Reset controller failed! (rc=%d)\n", rc);
2564                 goto fail;
2565         }
2566
2567         ahci_init_controller(host);
2568         ahci_print_info(host, "TEGRA-SATA");
2569         dev_dbg(dev, "controller init okay\n");
2570
2571 #if defined(TEGRA_AHCI_CONTEXT_RESTORE)
2572         /* Setup PG save/restore area: */
2573
2574         /* calculate the size */
2575         save_size = ARRAY_SIZE(pg_save_ipfs_registers) +
2576                             ARRAY_SIZE(pg_save_config_registers) +
2577                             ARRAY_SIZE(pg_save_bar5_registers) +
2578                             ARRAY_SIZE(pg_save_bar5_bkdr_registers);
2579
2580         /* and add save port_registers for all the ports */
2581         save_size += TEGRA_AHCI_NUM_PORTS *
2582                      (ARRAY_SIZE(pg_save_config_port_registers) +
2583                       ARRAY_SIZE(pg_save_bar5_port_registers) +
2584                       ARRAY_SIZE(pg_save_bar5_bkdr_port_registers));
2585
2586         /*
2587          * save_size is number of registers times number of bytes per
2588          * register to get total save size.
2589          */
2590         save_size *= sizeof(u32);
2591         tegra_hpriv->pg_save = devm_kzalloc(dev, save_size, GFP_KERNEL);
2592         if (!tegra_hpriv->pg_save) {
2593                 rc = -ENOMEM;
2594                 goto fail;
2595         }
2596 #endif
2597 #ifdef CONFIG_PM
2598 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
2599         tegra_hpriv->pg_state = SATA_ON;
2600
2601         /* setup sata idle timer */
2602         init_timer_deferrable(&tegra_hpriv->idle_timer);
2603         tegra_hpriv->idle_timer.function = tegra_ahci_idle_timer;
2604         tegra_hpriv->idle_timer.data = (unsigned long)host;
2605
2606         INIT_LIST_HEAD(&tegra_hpriv->qc_list);
2607
2608         /* use our own irq handler */
2609         irq_handler = tegra_ahci_interrupt;
2610 #endif
2611
2612 #endif
2613
2614         rc = ata_host_activate(host, irq_res->start, irq_handler, 0, &ahci_sht);
2615         if (rc == 0)
2616                 return 0;
2617
2618 fail:
2619         if (host) {
2620                 if (host->iomap[AHCI_PCI_BAR])
2621                         devm_iounmap(dev, host->iomap[AHCI_PCI_BAR]);
2622                 ata_host_detach(host);
2623         }
2624         if (hpriv)
2625                 devm_kfree(dev, hpriv);
2626
2627 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
2628         if (tegra_ahci_work_q)
2629                 destroy_workqueue(tegra_ahci_work_q);
2630 #endif
2631
2632         return rc;
2633 }
2634
2635 static int __init ahci_init(void)
2636 {
2637         return platform_driver_register(&tegra_platform_ahci_driver);
2638 }
2639
2640 static void __exit ahci_exit(void)
2641 {
2642         platform_driver_unregister(&tegra_platform_ahci_driver);
2643 }
2644
2645 #ifdef  CONFIG_DEBUG_FS
2646
2647 #include <linux/debugfs.h>
2648 #include <linux/seq_file.h>
2649
2650 static void dbg_ahci_dump_regs(struct seq_file *s, u32 *ptr, u32 base, u32 regs)
2651 {
2652 #define REGS_PER_LINE   4
2653
2654         u32 i, j;
2655         u32 lines = regs / REGS_PER_LINE;
2656
2657         for (i = 0; i < lines; i++) {
2658                 seq_printf(s, "0x%08x: ", base+(i*16));
2659                 for (j = 0; j < REGS_PER_LINE; ++j) {
2660                         seq_printf(s, "0x%08x ", readl(ptr));
2661                         ++ptr;
2662                 }
2663                 seq_puts(s, "\n");
2664         }
2665 #undef REGS_PER_LINE
2666 }
2667
2668 static int dbg_ahci_dump_show(struct seq_file *s, void *unused)
2669 {
2670         u32 base;
2671         u32 *ptr;
2672         u32 i;
2673
2674         if (!g_tegra_hpriv)
2675                 return 0;
2676
2677 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
2678                 tegra_ahci_runtime_resume(g_tegra_hpriv->dev);
2679 #endif
2680
2681         base = TEGRA_SATA_CONFIG_BASE;
2682         ptr = (u32 *)IO_TO_VIRT(base);
2683         seq_puts(s, "SATA CONFIG Registers:\n");
2684         seq_puts(s, "----------------------\n");
2685         dbg_ahci_dump_regs(s, ptr, base, 0x200);
2686
2687         base = TEGRA_SATA_BAR5_BASE;
2688         ptr = (u32 *)IO_TO_VIRT(base);
2689         seq_puts(s, "\nAHCI HBA Registers:\n");
2690         seq_puts(s, "-------------------\n");
2691         dbg_ahci_dump_regs(s, ptr, base, 64);
2692
2693         for (i = 0; i < TEGRA_AHCI_NUM_PORTS; ++i) {
2694                 base = TEGRA_SATA_BAR5_BASE + 0x100 + (0x80*i);
2695                 ptr = (u32 *)IO_TO_VIRT(base);
2696                 seq_printf(s, "\nPort %u Registers:\n", i);
2697                 seq_puts(s, "---------------\n");
2698                 dbg_ahci_dump_regs(s, ptr, base, 20);
2699         }
2700
2701 #ifdef  CONFIG_TEGRA_SATA_IDLE_POWERGATE
2702         /* adjust tegra_ahci_idle_time to minimum if it is too small */
2703         tegra_ahci_idle_time = max((u32)TEGRA_AHCI_MIN_IDLE_TIME,
2704                                    tegra_ahci_idle_time);
2705         seq_printf(s, "\nIdle Timeout = %u milli-seconds.\n",
2706                       tegra_ahci_idle_time);
2707 #endif
2708
2709         if (tegra_powergate_is_powered(TEGRA_POWERGATE_SATA))
2710                 seq_puts(s, "\n=== SATA controller is powered on ===\n\n");
2711         else
2712                 seq_puts(s, "\n=== SATA controller is powered off ===\n\n");
2713 #ifdef CONFIG_TEGRA_SATA_IDLE_POWERGATE
2714         tegra_ahci_runtime_suspend(g_tegra_hpriv->dev);
2715 #endif
2716
2717
2718         return 0;
2719 }
2720
2721 static int dbg_ahci_dump_open(struct inode *inode, struct file *file)
2722 {
2723         return single_open(file, dbg_ahci_dump_show, &inode->i_private);
2724 }
2725
2726 static const struct file_operations debug_fops = {
2727         .open           = dbg_ahci_dump_open,
2728         .read           = seq_read,
2729         .llseek         = seq_lseek,
2730         .release        = single_release,
2731 };
2732
2733 static int __init tegra_ahci_dump_debuginit(void)
2734 {
2735         (void) debugfs_create_file("tegra_ahci", S_IRUGO,
2736                                    NULL, NULL, &debug_fops);
2737 #ifdef  CONFIG_TEGRA_SATA_IDLE_POWERGATE
2738         (void) debugfs_create_u32("tegra_ahci_idle_ms", S_IRUGO | S_IXUGO
2739                                         | S_IWUSR | S_IWGRP,
2740                                    NULL, &tegra_ahci_idle_time);
2741 #endif
2742         return 0;
2743 }
2744 late_initcall(tegra_ahci_dump_debuginit);
2745 #endif
2746
2747 MODULE_AUTHOR("NVIDIA");
2748 MODULE_DESCRIPTION("Tegra AHCI SATA low-level driver");
2749 MODULE_LICENSE("GPL v2");
2750 MODULE_VERSION(DRV_VERSION);
2751
2752 module_init(ahci_init);
2753 module_exit(ahci_exit);