1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <linux/clk.h>
19#include <linux/delay.h>
20#include <linux/gpio/consumer.h>
21#include <linux/init.h>
22#include <linux/interrupt.h>
23#include <linux/iopoll.h>
24#include <linux/irq.h>
25#include <linux/irqchip/chained_irq.h>
26#include <linux/irqdomain.h>
27#include <linux/kernel.h>
28#include <linux/mfd/syscon.h>
29#include <linux/module.h>
30#include <linux/of_address.h>
31#include <linux/of_device.h>
32#include <linux/of_pci.h>
33#include <linux/of_platform.h>
34#include <linux/of_irq.h>
35#include <linux/pci.h>
36#include <linux/pci_ids.h>
37#include <linux/phy/phy.h>
38#include <linux/platform_device.h>
39#include <linux/reset.h>
40#include <linux/regmap.h>
41
42
43
44
45
46#define HIWORD_UPDATE(mask, val) (((mask) << 16) | (val))
47#define HIWORD_UPDATE_BIT(val) HIWORD_UPDATE(val, val)
48
49#define ENCODE_LANES(x) ((((x) >> 1) & 3) << 4)
50
51#define PCIE_CLIENT_BASE 0x0
52#define PCIE_CLIENT_CONFIG (PCIE_CLIENT_BASE + 0x00)
53#define PCIE_CLIENT_CONF_ENABLE HIWORD_UPDATE_BIT(0x0001)
54#define PCIE_CLIENT_LINK_TRAIN_ENABLE HIWORD_UPDATE_BIT(0x0002)
55#define PCIE_CLIENT_ARI_ENABLE HIWORD_UPDATE_BIT(0x0008)
56#define PCIE_CLIENT_CONF_LANE_NUM(x) HIWORD_UPDATE(0x0030, ENCODE_LANES(x))
57#define PCIE_CLIENT_MODE_RC HIWORD_UPDATE_BIT(0x0040)
58#define PCIE_CLIENT_GEN_SEL_1 HIWORD_UPDATE(0x0080, 0)
59#define PCIE_CLIENT_GEN_SEL_2 HIWORD_UPDATE_BIT(0x0080)
60#define PCIE_CLIENT_DEBUG_OUT_0 (PCIE_CLIENT_BASE + 0x3c)
61#define PCIE_CLIENT_DEBUG_LTSSM_MASK GENMASK(5, 0)
62#define PCIE_CLIENT_DEBUG_LTSSM_L1 0x18
63#define PCIE_CLIENT_DEBUG_LTSSM_L2 0x19
64#define PCIE_CLIENT_BASIC_STATUS1 (PCIE_CLIENT_BASE + 0x48)
65#define PCIE_CLIENT_LINK_STATUS_UP 0x00300000
66#define PCIE_CLIENT_LINK_STATUS_MASK 0x00300000
67#define PCIE_CLIENT_INT_MASK (PCIE_CLIENT_BASE + 0x4c)
68#define PCIE_CLIENT_INT_STATUS (PCIE_CLIENT_BASE + 0x50)
69#define PCIE_CLIENT_INTR_MASK GENMASK(8, 5)
70#define PCIE_CLIENT_INTR_SHIFT 5
71#define PCIE_CLIENT_INT_LEGACY_DONE BIT(15)
72#define PCIE_CLIENT_INT_MSG BIT(14)
73#define PCIE_CLIENT_INT_HOT_RST BIT(13)
74#define PCIE_CLIENT_INT_DPA BIT(12)
75#define PCIE_CLIENT_INT_FATAL_ERR BIT(11)
76#define PCIE_CLIENT_INT_NFATAL_ERR BIT(10)
77#define PCIE_CLIENT_INT_CORR_ERR BIT(9)
78#define PCIE_CLIENT_INT_INTD BIT(8)
79#define PCIE_CLIENT_INT_INTC BIT(7)
80#define PCIE_CLIENT_INT_INTB BIT(6)
81#define PCIE_CLIENT_INT_INTA BIT(5)
82#define PCIE_CLIENT_INT_LOCAL BIT(4)
83#define PCIE_CLIENT_INT_UDMA BIT(3)
84#define PCIE_CLIENT_INT_PHY BIT(2)
85#define PCIE_CLIENT_INT_HOT_PLUG BIT(1)
86#define PCIE_CLIENT_INT_PWR_STCG BIT(0)
87
88#define PCIE_CLIENT_INT_LEGACY \
89 (PCIE_CLIENT_INT_INTA | PCIE_CLIENT_INT_INTB | \
90 PCIE_CLIENT_INT_INTC | PCIE_CLIENT_INT_INTD)
91
92#define PCIE_CLIENT_INT_CLI \
93 (PCIE_CLIENT_INT_CORR_ERR | PCIE_CLIENT_INT_NFATAL_ERR | \
94 PCIE_CLIENT_INT_FATAL_ERR | PCIE_CLIENT_INT_DPA | \
95 PCIE_CLIENT_INT_HOT_RST | PCIE_CLIENT_INT_MSG | \
96 PCIE_CLIENT_INT_LEGACY_DONE | PCIE_CLIENT_INT_LEGACY | \
97 PCIE_CLIENT_INT_PHY)
98
99#define PCIE_CORE_CTRL_MGMT_BASE 0x900000
100#define PCIE_CORE_CTRL (PCIE_CORE_CTRL_MGMT_BASE + 0x000)
101#define PCIE_CORE_PL_CONF_SPEED_5G 0x00000008
102#define PCIE_CORE_PL_CONF_SPEED_MASK 0x00000018
103#define PCIE_CORE_PL_CONF_LANE_MASK 0x00000006
104#define PCIE_CORE_PL_CONF_LANE_SHIFT 1
105#define PCIE_CORE_CTRL_PLC1 (PCIE_CORE_CTRL_MGMT_BASE + 0x004)
106#define PCIE_CORE_CTRL_PLC1_FTS_MASK GENMASK(23, 8)
107#define PCIE_CORE_CTRL_PLC1_FTS_SHIFT 8
108#define PCIE_CORE_CTRL_PLC1_FTS_CNT 0xffff
109#define PCIE_CORE_TXCREDIT_CFG1 (PCIE_CORE_CTRL_MGMT_BASE + 0x020)
110#define PCIE_CORE_TXCREDIT_CFG1_MUI_MASK 0xFFFF0000
111#define PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT 16
112#define PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(x) \
113 (((x) >> 3) << PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT)
114#define PCIE_CORE_INT_STATUS (PCIE_CORE_CTRL_MGMT_BASE + 0x20c)
115#define PCIE_CORE_INT_PRFPE BIT(0)
116#define PCIE_CORE_INT_CRFPE BIT(1)
117#define PCIE_CORE_INT_RRPE BIT(2)
118#define PCIE_CORE_INT_PRFO BIT(3)
119#define PCIE_CORE_INT_CRFO BIT(4)
120#define PCIE_CORE_INT_RT BIT(5)
121#define PCIE_CORE_INT_RTR BIT(6)
122#define PCIE_CORE_INT_PE BIT(7)
123#define PCIE_CORE_INT_MTR BIT(8)
124#define PCIE_CORE_INT_UCR BIT(9)
125#define PCIE_CORE_INT_FCE BIT(10)
126#define PCIE_CORE_INT_CT BIT(11)
127#define PCIE_CORE_INT_UTC BIT(18)
128#define PCIE_CORE_INT_MMVC BIT(19)
129#define PCIE_CORE_CONFIG_VENDOR (PCIE_CORE_CTRL_MGMT_BASE + 0x44)
130#define PCIE_CORE_INT_MASK (PCIE_CORE_CTRL_MGMT_BASE + 0x210)
131#define PCIE_RC_BAR_CONF (PCIE_CORE_CTRL_MGMT_BASE + 0x300)
132
133#define PCIE_CORE_INT \
134 (PCIE_CORE_INT_PRFPE | PCIE_CORE_INT_CRFPE | \
135 PCIE_CORE_INT_RRPE | PCIE_CORE_INT_CRFO | \
136 PCIE_CORE_INT_RT | PCIE_CORE_INT_RTR | \
137 PCIE_CORE_INT_PE | PCIE_CORE_INT_MTR | \
138 PCIE_CORE_INT_UCR | PCIE_CORE_INT_FCE | \
139 PCIE_CORE_INT_CT | PCIE_CORE_INT_UTC | \
140 PCIE_CORE_INT_MMVC)
141
142#define PCIE_RC_CONFIG_BASE 0xa00000
143#define PCIE_RC_CONFIG_RID_CCR (PCIE_RC_CONFIG_BASE + 0x08)
144#define PCIE_RC_CONFIG_SCC_SHIFT 16
145#define PCIE_RC_CONFIG_DCR (PCIE_RC_CONFIG_BASE + 0xc4)
146#define PCIE_RC_CONFIG_DCR_CSPL_SHIFT 18
147#define PCIE_RC_CONFIG_DCR_CSPL_LIMIT 0xff
148#define PCIE_RC_CONFIG_DCR_CPLS_SHIFT 26
149#define PCIE_RC_CONFIG_LINK_CAP (PCIE_RC_CONFIG_BASE + 0xcc)
150#define PCIE_RC_CONFIG_LINK_CAP_L0S BIT(10)
151#define PCIE_RC_CONFIG_LCS (PCIE_RC_CONFIG_BASE + 0xd0)
152#define PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2 (PCIE_RC_CONFIG_BASE + 0x90c)
153#define PCIE_RC_CONFIG_THP_CAP (PCIE_RC_CONFIG_BASE + 0x274)
154#define PCIE_RC_CONFIG_THP_CAP_NEXT_MASK GENMASK(31, 20)
155
156#define PCIE_CORE_AXI_CONF_BASE 0xc00000
157#define PCIE_CORE_OB_REGION_ADDR0 (PCIE_CORE_AXI_CONF_BASE + 0x0)
158#define PCIE_CORE_OB_REGION_ADDR0_NUM_BITS 0x3f
159#define PCIE_CORE_OB_REGION_ADDR0_LO_ADDR 0xffffff00
160#define PCIE_CORE_OB_REGION_ADDR1 (PCIE_CORE_AXI_CONF_BASE + 0x4)
161#define PCIE_CORE_OB_REGION_DESC0 (PCIE_CORE_AXI_CONF_BASE + 0x8)
162#define PCIE_CORE_OB_REGION_DESC1 (PCIE_CORE_AXI_CONF_BASE + 0xc)
163
164#define PCIE_CORE_AXI_INBOUND_BASE 0xc00800
165#define PCIE_RP_IB_ADDR0 (PCIE_CORE_AXI_INBOUND_BASE + 0x0)
166#define PCIE_CORE_IB_REGION_ADDR0_NUM_BITS 0x3f
167#define PCIE_CORE_IB_REGION_ADDR0_LO_ADDR 0xffffff00
168#define PCIE_RP_IB_ADDR1 (PCIE_CORE_AXI_INBOUND_BASE + 0x4)
169
170
171#define AXI_REGION_SIZE BIT(20)
172
173#define AXI_REGION_0_SIZE (32 * (0x1 << 20))
174#define OB_REG_SIZE_SHIFT 5
175#define IB_ROOT_PORT_REG_SIZE_SHIFT 3
176#define AXI_WRAPPER_IO_WRITE 0x6
177#define AXI_WRAPPER_MEM_WRITE 0x2
178#define AXI_WRAPPER_NOR_MSG 0xc
179
180#define MAX_AXI_IB_ROOTPORT_REGION_NUM 3
181#define MIN_AXI_ADDR_BITS_PASSED 8
182#define PCIE_RC_SEND_PME_OFF 0x11960
183#define ROCKCHIP_VENDOR_ID 0x1d87
184#define PCIE_ECAM_BUS(x) (((x) & 0xff) << 20)
185#define PCIE_ECAM_DEV(x) (((x) & 0x1f) << 15)
186#define PCIE_ECAM_FUNC(x) (((x) & 0x7) << 12)
187#define PCIE_ECAM_REG(x) (((x) & 0xfff) << 0)
188#define PCIE_ECAM_ADDR(bus, dev, func, reg) \
189 (PCIE_ECAM_BUS(bus) | PCIE_ECAM_DEV(dev) | \
190 PCIE_ECAM_FUNC(func) | PCIE_ECAM_REG(reg))
191#define PCIE_LINK_IS_L2(x) \
192 (((x) & PCIE_CLIENT_DEBUG_LTSSM_MASK) == PCIE_CLIENT_DEBUG_LTSSM_L2)
193#define PCIE_LINK_UP(x) \
194 (((x) & PCIE_CLIENT_LINK_STATUS_MASK) == PCIE_CLIENT_LINK_STATUS_UP)
195#define PCIE_LINK_IS_GEN2(x) \
196 (((x) & PCIE_CORE_PL_CONF_SPEED_MASK) == PCIE_CORE_PL_CONF_SPEED_5G)
197
198#define RC_REGION_0_ADDR_TRANS_H 0x00000000
199#define RC_REGION_0_ADDR_TRANS_L 0x00000000
200#define RC_REGION_0_PASS_BITS (25 - 1)
201#define MAX_AXI_WRAPPER_REGION_NUM 33
202
203struct rockchip_pcie {
204 void __iomem *reg_base;
205 void __iomem *apb_base;
206 struct phy *phy;
207 struct reset_control *core_rst;
208 struct reset_control *mgmt_rst;
209 struct reset_control *mgmt_sticky_rst;
210 struct reset_control *pipe_rst;
211 struct reset_control *pm_rst;
212 struct reset_control *aclk_rst;
213 struct reset_control *pclk_rst;
214 struct clk *aclk_pcie;
215 struct clk *aclk_perf_pcie;
216 struct clk *hclk_pcie;
217 struct clk *clk_pcie_pm;
218 struct regulator *vpcie3v3;
219 struct regulator *vpcie1v8;
220 struct regulator *vpcie0v9;
221 struct gpio_desc *ep_gpio;
222 u32 lanes;
223 u8 root_bus_nr;
224 int link_gen;
225 struct device *dev;
226 struct irq_domain *irq_domain;
227 int offset;
228 struct pci_bus *root_bus;
229 struct resource *io;
230 phys_addr_t io_bus_addr;
231 u32 io_size;
232 void __iomem *msg_region;
233 u32 mem_size;
234 phys_addr_t msg_bus_addr;
235 phys_addr_t mem_bus_addr;
236};
237
238static u32 rockchip_pcie_read(struct rockchip_pcie *rockchip, u32 reg)
239{
240 return readl(rockchip->apb_base + reg);
241}
242
243static void rockchip_pcie_write(struct rockchip_pcie *rockchip, u32 val,
244 u32 reg)
245{
246 writel(val, rockchip->apb_base + reg);
247}
248
249static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip)
250{
251 u32 status;
252
253 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
254 status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE);
255 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
256}
257
258static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip)
259{
260 u32 status;
261
262 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
263 status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16;
264 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
265}
266
267static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip)
268{
269 u32 val;
270
271
272 val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1);
273 val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK;
274 val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000);
275 rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1);
276}
277
278static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip,
279 struct pci_bus *bus, int dev)
280{
281
282 if (bus->number == rockchip->root_bus_nr && dev > 0)
283 return 0;
284
285
286
287
288
289 if (bus->primary == rockchip->root_bus_nr && dev > 0)
290 return 0;
291
292 return 1;
293}
294
295static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip,
296 int where, int size, u32 *val)
297{
298 void __iomem *addr = rockchip->apb_base + PCIE_RC_CONFIG_BASE + where;
299
300 if (!IS_ALIGNED((uintptr_t)addr, size)) {
301 *val = 0;
302 return PCIBIOS_BAD_REGISTER_NUMBER;
303 }
304
305 if (size == 4) {
306 *val = readl(addr);
307 } else if (size == 2) {
308 *val = readw(addr);
309 } else if (size == 1) {
310 *val = readb(addr);
311 } else {
312 *val = 0;
313 return PCIBIOS_BAD_REGISTER_NUMBER;
314 }
315 return PCIBIOS_SUCCESSFUL;
316}
317
318static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip,
319 int where, int size, u32 val)
320{
321 u32 mask, tmp, offset;
322
323 offset = where & ~0x3;
324
325 if (size == 4) {
326 writel(val, rockchip->apb_base + PCIE_RC_CONFIG_BASE + offset);
327 return PCIBIOS_SUCCESSFUL;
328 }
329
330 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
331
332
333
334
335
336
337 tmp = readl(rockchip->apb_base + PCIE_RC_CONFIG_BASE + offset) & mask;
338 tmp |= val << ((where & 0x3) * 8);
339 writel(tmp, rockchip->apb_base + PCIE_RC_CONFIG_BASE + offset);
340
341 return PCIBIOS_SUCCESSFUL;
342}
343
344static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip,
345 struct pci_bus *bus, u32 devfn,
346 int where, int size, u32 *val)
347{
348 u32 busdev;
349
350 busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
351 PCI_FUNC(devfn), where);
352
353 if (!IS_ALIGNED(busdev, size)) {
354 *val = 0;
355 return PCIBIOS_BAD_REGISTER_NUMBER;
356 }
357
358 if (size == 4) {
359 *val = readl(rockchip->reg_base + busdev);
360 } else if (size == 2) {
361 *val = readw(rockchip->reg_base + busdev);
362 } else if (size == 1) {
363 *val = readb(rockchip->reg_base + busdev);
364 } else {
365 *val = 0;
366 return PCIBIOS_BAD_REGISTER_NUMBER;
367 }
368 return PCIBIOS_SUCCESSFUL;
369}
370
371static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip,
372 struct pci_bus *bus, u32 devfn,
373 int where, int size, u32 val)
374{
375 u32 busdev;
376
377 busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
378 PCI_FUNC(devfn), where);
379 if (!IS_ALIGNED(busdev, size))
380 return PCIBIOS_BAD_REGISTER_NUMBER;
381
382 if (size == 4)
383 writel(val, rockchip->reg_base + busdev);
384 else if (size == 2)
385 writew(val, rockchip->reg_base + busdev);
386 else if (size == 1)
387 writeb(val, rockchip->reg_base + busdev);
388 else
389 return PCIBIOS_BAD_REGISTER_NUMBER;
390
391 return PCIBIOS_SUCCESSFUL;
392}
393
394static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,
395 int size, u32 *val)
396{
397 struct rockchip_pcie *rockchip = bus->sysdata;
398
399 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) {
400 *val = 0xffffffff;
401 return PCIBIOS_DEVICE_NOT_FOUND;
402 }
403
404 if (bus->number == rockchip->root_bus_nr)
405 return rockchip_pcie_rd_own_conf(rockchip, where, size, val);
406
407 return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, val);
408}
409
410static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
411 int where, int size, u32 val)
412{
413 struct rockchip_pcie *rockchip = bus->sysdata;
414
415 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))
416 return PCIBIOS_DEVICE_NOT_FOUND;
417
418 if (bus->number == rockchip->root_bus_nr)
419 return rockchip_pcie_wr_own_conf(rockchip, where, size, val);
420
421 return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, val);
422}
423
424static struct pci_ops rockchip_pcie_ops = {
425 .read = rockchip_pcie_rd_conf,
426 .write = rockchip_pcie_wr_conf,
427};
428
429static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip)
430{
431 int curr;
432 u32 status, scale, power;
433
434 if (IS_ERR(rockchip->vpcie3v3))
435 return;
436
437
438
439
440
441
442
443 curr = regulator_get_current_limit(rockchip->vpcie3v3);
444 if (curr <= 0)
445 return;
446
447 scale = 3;
448 curr = curr / 1000;
449 power = (curr * 3300) / 1000;
450 while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) {
451 if (!scale) {
452 dev_warn(rockchip->dev, "invalid power supply\n");
453 return;
454 }
455 scale--;
456 power = power / 10;
457 }
458
459 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR);
460 status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) |
461 (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT);
462 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR);
463}
464
465
466
467
468
469static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
470{
471 struct device *dev = rockchip->dev;
472 int err;
473 u32 status;
474
475 gpiod_set_value(rockchip->ep_gpio, 0);
476
477 err = reset_control_assert(rockchip->aclk_rst);
478 if (err) {
479 dev_err(dev, "assert aclk_rst err %d\n", err);
480 return err;
481 }
482
483 err = reset_control_assert(rockchip->pclk_rst);
484 if (err) {
485 dev_err(dev, "assert pclk_rst err %d\n", err);
486 return err;
487 }
488
489 err = reset_control_assert(rockchip->pm_rst);
490 if (err) {
491 dev_err(dev, "assert pm_rst err %d\n", err);
492 return err;
493 }
494
495 err = phy_init(rockchip->phy);
496 if (err < 0) {
497 dev_err(dev, "fail to init phy, err %d\n", err);
498 return err;
499 }
500
501 err = reset_control_assert(rockchip->core_rst);
502 if (err) {
503 dev_err(dev, "assert core_rst err %d\n", err);
504 return err;
505 }
506
507 err = reset_control_assert(rockchip->mgmt_rst);
508 if (err) {
509 dev_err(dev, "assert mgmt_rst err %d\n", err);
510 return err;
511 }
512
513 err = reset_control_assert(rockchip->mgmt_sticky_rst);
514 if (err) {
515 dev_err(dev, "assert mgmt_sticky_rst err %d\n", err);
516 return err;
517 }
518
519 err = reset_control_assert(rockchip->pipe_rst);
520 if (err) {
521 dev_err(dev, "assert pipe_rst err %d\n", err);
522 return err;
523 }
524
525 udelay(10);
526
527 err = reset_control_deassert(rockchip->pm_rst);
528 if (err) {
529 dev_err(dev, "deassert pm_rst err %d\n", err);
530 return err;
531 }
532
533 err = reset_control_deassert(rockchip->aclk_rst);
534 if (err) {
535 dev_err(dev, "deassert aclk_rst err %d\n", err);
536 return err;
537 }
538
539 err = reset_control_deassert(rockchip->pclk_rst);
540 if (err) {
541 dev_err(dev, "deassert pclk_rst err %d\n", err);
542 return err;
543 }
544
545 if (rockchip->link_gen == 2)
546 rockchip_pcie_write(rockchip, PCIE_CLIENT_GEN_SEL_2,
547 PCIE_CLIENT_CONFIG);
548 else
549 rockchip_pcie_write(rockchip, PCIE_CLIENT_GEN_SEL_1,
550 PCIE_CLIENT_CONFIG);
551
552 rockchip_pcie_write(rockchip,
553 PCIE_CLIENT_CONF_ENABLE |
554 PCIE_CLIENT_LINK_TRAIN_ENABLE |
555 PCIE_CLIENT_ARI_ENABLE |
556 PCIE_CLIENT_CONF_LANE_NUM(rockchip->lanes) |
557 PCIE_CLIENT_MODE_RC,
558 PCIE_CLIENT_CONFIG);
559
560 err = phy_power_on(rockchip->phy);
561 if (err) {
562 dev_err(dev, "fail to power on phy, err %d\n", err);
563 return err;
564 }
565
566
567
568
569
570 err = reset_control_deassert(rockchip->mgmt_sticky_rst);
571 if (err) {
572 dev_err(dev, "deassert mgmt_sticky_rst err %d\n", err);
573 return err;
574 }
575
576 err = reset_control_deassert(rockchip->core_rst);
577 if (err) {
578 dev_err(dev, "deassert core_rst err %d\n", err);
579 return err;
580 }
581
582 err = reset_control_deassert(rockchip->mgmt_rst);
583 if (err) {
584 dev_err(dev, "deassert mgmt_rst err %d\n", err);
585 return err;
586 }
587
588 err = reset_control_deassert(rockchip->pipe_rst);
589 if (err) {
590 dev_err(dev, "deassert pipe_rst err %d\n", err);
591 return err;
592 }
593
594
595 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1);
596 status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) |
597 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT);
598 rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1);
599
600 rockchip_pcie_set_power_limit(rockchip);
601
602
603 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
604 status |= PCI_EXP_LNKSTA_SLC << 16;
605 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
606
607
608 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
609 status |= PCI_EXP_LNKCTL_RCB;
610 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
611
612
613 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,
614 PCIE_CLIENT_CONFIG);
615
616 gpiod_set_value(rockchip->ep_gpio, 1);
617
618
619 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1,
620 status, PCIE_LINK_UP(status), 20,
621 500 * USEC_PER_MSEC);
622 if (err) {
623 dev_err(dev, "PCIe link training gen1 timeout!\n");
624 return -ETIMEDOUT;
625 }
626
627 if (rockchip->link_gen == 2) {
628
629
630
631
632 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
633 status |= PCI_EXP_LNKCTL_RL;
634 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
635
636 err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL,
637 status, PCIE_LINK_IS_GEN2(status), 20,
638 500 * USEC_PER_MSEC);
639 if (err)
640 dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n");
641 }
642
643
644 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
645 status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >>
646 PCIE_CORE_PL_CONF_LANE_SHIFT);
647 dev_dbg(dev, "current link width is x%d\n", status);
648
649 rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID,
650 PCIE_CORE_CONFIG_VENDOR);
651 rockchip_pcie_write(rockchip,
652 PCI_CLASS_BRIDGE_PCI << PCIE_RC_CONFIG_SCC_SHIFT,
653 PCIE_RC_CONFIG_RID_CCR);
654
655
656 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP);
657 status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
658 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP);
659
660
661 if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) {
662 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP);
663 status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S;
664 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP);
665 }
666
667 rockchip_pcie_write(rockchip, 0x0, PCIE_RC_BAR_CONF);
668
669 rockchip_pcie_write(rockchip,
670 (RC_REGION_0_ADDR_TRANS_L + RC_REGION_0_PASS_BITS),
671 PCIE_CORE_OB_REGION_ADDR0);
672 rockchip_pcie_write(rockchip, RC_REGION_0_ADDR_TRANS_H,
673 PCIE_CORE_OB_REGION_ADDR1);
674 rockchip_pcie_write(rockchip, 0x0080000a, PCIE_CORE_OB_REGION_DESC0);
675 rockchip_pcie_write(rockchip, 0x0, PCIE_CORE_OB_REGION_DESC1);
676
677 return 0;
678}
679
680static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg)
681{
682 struct rockchip_pcie *rockchip = arg;
683 struct device *dev = rockchip->dev;
684 u32 reg;
685 u32 sub_reg;
686
687 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
688 if (reg & PCIE_CLIENT_INT_LOCAL) {
689 dev_dbg(dev, "local interrupt received\n");
690 sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS);
691 if (sub_reg & PCIE_CORE_INT_PRFPE)
692 dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n");
693
694 if (sub_reg & PCIE_CORE_INT_CRFPE)
695 dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n");
696
697 if (sub_reg & PCIE_CORE_INT_RRPE)
698 dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n");
699
700 if (sub_reg & PCIE_CORE_INT_PRFO)
701 dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n");
702
703 if (sub_reg & PCIE_CORE_INT_CRFO)
704 dev_dbg(dev, "overflow occurred in the completion receive FIFO\n");
705
706 if (sub_reg & PCIE_CORE_INT_RT)
707 dev_dbg(dev, "replay timer timed out\n");
708
709 if (sub_reg & PCIE_CORE_INT_RTR)
710 dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n");
711
712 if (sub_reg & PCIE_CORE_INT_PE)
713 dev_dbg(dev, "phy error detected on receive side\n");
714
715 if (sub_reg & PCIE_CORE_INT_MTR)
716 dev_dbg(dev, "malformed TLP received from the link\n");
717
718 if (sub_reg & PCIE_CORE_INT_UCR)
719 dev_dbg(dev, "malformed TLP received from the link\n");
720
721 if (sub_reg & PCIE_CORE_INT_FCE)
722 dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n");
723
724 if (sub_reg & PCIE_CORE_INT_CT)
725 dev_dbg(dev, "a request timed out waiting for completion\n");
726
727 if (sub_reg & PCIE_CORE_INT_UTC)
728 dev_dbg(dev, "unmapped TC error\n");
729
730 if (sub_reg & PCIE_CORE_INT_MMVC)
731 dev_dbg(dev, "MSI mask register changes\n");
732
733 rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS);
734 } else if (reg & PCIE_CLIENT_INT_PHY) {
735 dev_dbg(dev, "phy link changes\n");
736 rockchip_pcie_update_txcredit_mui(rockchip);
737 rockchip_pcie_clr_bw_int(rockchip);
738 }
739
740 rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL,
741 PCIE_CLIENT_INT_STATUS);
742
743 return IRQ_HANDLED;
744}
745
746static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg)
747{
748 struct rockchip_pcie *rockchip = arg;
749 struct device *dev = rockchip->dev;
750 u32 reg;
751
752 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
753 if (reg & PCIE_CLIENT_INT_LEGACY_DONE)
754 dev_dbg(dev, "legacy done interrupt received\n");
755
756 if (reg & PCIE_CLIENT_INT_MSG)
757 dev_dbg(dev, "message done interrupt received\n");
758
759 if (reg & PCIE_CLIENT_INT_HOT_RST)
760 dev_dbg(dev, "hot reset interrupt received\n");
761
762 if (reg & PCIE_CLIENT_INT_DPA)
763 dev_dbg(dev, "dpa interrupt received\n");
764
765 if (reg & PCIE_CLIENT_INT_FATAL_ERR)
766 dev_dbg(dev, "fatal error interrupt received\n");
767
768 if (reg & PCIE_CLIENT_INT_NFATAL_ERR)
769 dev_dbg(dev, "no fatal error interrupt received\n");
770
771 if (reg & PCIE_CLIENT_INT_CORR_ERR)
772 dev_dbg(dev, "correctable error interrupt received\n");
773
774 if (reg & PCIE_CLIENT_INT_PHY)
775 dev_dbg(dev, "phy interrupt received\n");
776
777 rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE |
778 PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST |
779 PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR |
780 PCIE_CLIENT_INT_NFATAL_ERR |
781 PCIE_CLIENT_INT_CORR_ERR |
782 PCIE_CLIENT_INT_PHY),
783 PCIE_CLIENT_INT_STATUS);
784
785 return IRQ_HANDLED;
786}
787
788static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc)
789{
790 struct irq_chip *chip = irq_desc_get_chip(desc);
791 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);
792 struct device *dev = rockchip->dev;
793 u32 reg;
794 u32 hwirq;
795 u32 virq;
796
797 chained_irq_enter(chip, desc);
798
799 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
800 reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT;
801
802 while (reg) {
803 hwirq = ffs(reg) - 1;
804 reg &= ~BIT(hwirq);
805
806 virq = irq_find_mapping(rockchip->irq_domain, hwirq);
807 if (virq)
808 generic_handle_irq(virq);
809 else
810 dev_err(dev, "unexpected IRQ, INT%d\n", hwirq);
811 }
812
813 chained_irq_exit(chip, desc);
814}
815
816
817
818
819
820
821
822
823static int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip)
824{
825 struct device *dev = rockchip->dev;
826 struct platform_device *pdev = to_platform_device(dev);
827 struct device_node *node = dev->of_node;
828 struct resource *regs;
829 int irq;
830 int err;
831
832 regs = platform_get_resource_byname(pdev,
833 IORESOURCE_MEM,
834 "axi-base");
835 rockchip->reg_base = devm_pci_remap_cfg_resource(dev, regs);
836 if (IS_ERR(rockchip->reg_base))
837 return PTR_ERR(rockchip->reg_base);
838
839 regs = platform_get_resource_byname(pdev,
840 IORESOURCE_MEM,
841 "apb-base");
842 rockchip->apb_base = devm_ioremap_resource(dev, regs);
843 if (IS_ERR(rockchip->apb_base))
844 return PTR_ERR(rockchip->apb_base);
845
846 rockchip->phy = devm_phy_get(dev, "pcie-phy");
847 if (IS_ERR(rockchip->phy)) {
848 if (PTR_ERR(rockchip->phy) != -EPROBE_DEFER)
849 dev_err(dev, "missing phy\n");
850 return PTR_ERR(rockchip->phy);
851 }
852
853 rockchip->lanes = 1;
854 err = of_property_read_u32(node, "num-lanes", &rockchip->lanes);
855 if (!err && (rockchip->lanes == 0 ||
856 rockchip->lanes == 3 ||
857 rockchip->lanes > 4)) {
858 dev_warn(dev, "invalid num-lanes, default to use one lane\n");
859 rockchip->lanes = 1;
860 }
861
862 rockchip->link_gen = of_pci_get_max_link_speed(node);
863 if (rockchip->link_gen < 0 || rockchip->link_gen > 2)
864 rockchip->link_gen = 2;
865
866 rockchip->core_rst = devm_reset_control_get(dev, "core");
867 if (IS_ERR(rockchip->core_rst)) {
868 if (PTR_ERR(rockchip->core_rst) != -EPROBE_DEFER)
869 dev_err(dev, "missing core reset property in node\n");
870 return PTR_ERR(rockchip->core_rst);
871 }
872
873 rockchip->mgmt_rst = devm_reset_control_get(dev, "mgmt");
874 if (IS_ERR(rockchip->mgmt_rst)) {
875 if (PTR_ERR(rockchip->mgmt_rst) != -EPROBE_DEFER)
876 dev_err(dev, "missing mgmt reset property in node\n");
877 return PTR_ERR(rockchip->mgmt_rst);
878 }
879
880 rockchip->mgmt_sticky_rst = devm_reset_control_get(dev, "mgmt-sticky");
881 if (IS_ERR(rockchip->mgmt_sticky_rst)) {
882 if (PTR_ERR(rockchip->mgmt_sticky_rst) != -EPROBE_DEFER)
883 dev_err(dev, "missing mgmt-sticky reset property in node\n");
884 return PTR_ERR(rockchip->mgmt_sticky_rst);
885 }
886
887 rockchip->pipe_rst = devm_reset_control_get(dev, "pipe");
888 if (IS_ERR(rockchip->pipe_rst)) {
889 if (PTR_ERR(rockchip->pipe_rst) != -EPROBE_DEFER)
890 dev_err(dev, "missing pipe reset property in node\n");
891 return PTR_ERR(rockchip->pipe_rst);
892 }
893
894 rockchip->pm_rst = devm_reset_control_get(dev, "pm");
895 if (IS_ERR(rockchip->pm_rst)) {
896 if (PTR_ERR(rockchip->pm_rst) != -EPROBE_DEFER)
897 dev_err(dev, "missing pm reset property in node\n");
898 return PTR_ERR(rockchip->pm_rst);
899 }
900
901 rockchip->pclk_rst = devm_reset_control_get(dev, "pclk");
902 if (IS_ERR(rockchip->pclk_rst)) {
903 if (PTR_ERR(rockchip->pclk_rst) != -EPROBE_DEFER)
904 dev_err(dev, "missing pclk reset property in node\n");
905 return PTR_ERR(rockchip->pclk_rst);
906 }
907
908 rockchip->aclk_rst = devm_reset_control_get(dev, "aclk");
909 if (IS_ERR(rockchip->aclk_rst)) {
910 if (PTR_ERR(rockchip->aclk_rst) != -EPROBE_DEFER)
911 dev_err(dev, "missing aclk reset property in node\n");
912 return PTR_ERR(rockchip->aclk_rst);
913 }
914
915 rockchip->ep_gpio = devm_gpiod_get(dev, "ep", GPIOD_OUT_HIGH);
916 if (IS_ERR(rockchip->ep_gpio)) {
917 dev_err(dev, "missing ep-gpios property in node\n");
918 return PTR_ERR(rockchip->ep_gpio);
919 }
920
921 rockchip->aclk_pcie = devm_clk_get(dev, "aclk");
922 if (IS_ERR(rockchip->aclk_pcie)) {
923 dev_err(dev, "aclk clock not found\n");
924 return PTR_ERR(rockchip->aclk_pcie);
925 }
926
927 rockchip->aclk_perf_pcie = devm_clk_get(dev, "aclk-perf");
928 if (IS_ERR(rockchip->aclk_perf_pcie)) {
929 dev_err(dev, "aclk_perf clock not found\n");
930 return PTR_ERR(rockchip->aclk_perf_pcie);
931 }
932
933 rockchip->hclk_pcie = devm_clk_get(dev, "hclk");
934 if (IS_ERR(rockchip->hclk_pcie)) {
935 dev_err(dev, "hclk clock not found\n");
936 return PTR_ERR(rockchip->hclk_pcie);
937 }
938
939 rockchip->clk_pcie_pm = devm_clk_get(dev, "pm");
940 if (IS_ERR(rockchip->clk_pcie_pm)) {
941 dev_err(dev, "pm clock not found\n");
942 return PTR_ERR(rockchip->clk_pcie_pm);
943 }
944
945 irq = platform_get_irq_byname(pdev, "sys");
946 if (irq < 0) {
947 dev_err(dev, "missing sys IRQ resource\n");
948 return -EINVAL;
949 }
950
951 err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler,
952 IRQF_SHARED, "pcie-sys", rockchip);
953 if (err) {
954 dev_err(dev, "failed to request PCIe subsystem IRQ\n");
955 return err;
956 }
957
958 irq = platform_get_irq_byname(pdev, "legacy");
959 if (irq < 0) {
960 dev_err(dev, "missing legacy IRQ resource\n");
961 return -EINVAL;
962 }
963
964 irq_set_chained_handler_and_data(irq,
965 rockchip_pcie_legacy_int_handler,
966 rockchip);
967
968 irq = platform_get_irq_byname(pdev, "client");
969 if (irq < 0) {
970 dev_err(dev, "missing client IRQ resource\n");
971 return -EINVAL;
972 }
973
974 err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler,
975 IRQF_SHARED, "pcie-client", rockchip);
976 if (err) {
977 dev_err(dev, "failed to request PCIe client IRQ\n");
978 return err;
979 }
980
981 rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3");
982 if (IS_ERR(rockchip->vpcie3v3)) {
983 if (PTR_ERR(rockchip->vpcie3v3) == -EPROBE_DEFER)
984 return -EPROBE_DEFER;
985 dev_info(dev, "no vpcie3v3 regulator found\n");
986 }
987
988 rockchip->vpcie1v8 = devm_regulator_get_optional(dev, "vpcie1v8");
989 if (IS_ERR(rockchip->vpcie1v8)) {
990 if (PTR_ERR(rockchip->vpcie1v8) == -EPROBE_DEFER)
991 return -EPROBE_DEFER;
992 dev_info(dev, "no vpcie1v8 regulator found\n");
993 }
994
995 rockchip->vpcie0v9 = devm_regulator_get_optional(dev, "vpcie0v9");
996 if (IS_ERR(rockchip->vpcie0v9)) {
997 if (PTR_ERR(rockchip->vpcie0v9) == -EPROBE_DEFER)
998 return -EPROBE_DEFER;
999 dev_info(dev, "no vpcie0v9 regulator found\n");
1000 }
1001
1002 return 0;
1003}
1004
1005static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip)
1006{
1007 struct device *dev = rockchip->dev;
1008 int err;
1009
1010 if (!IS_ERR(rockchip->vpcie3v3)) {
1011 err = regulator_enable(rockchip->vpcie3v3);
1012 if (err) {
1013 dev_err(dev, "fail to enable vpcie3v3 regulator\n");
1014 goto err_out;
1015 }
1016 }
1017
1018 if (!IS_ERR(rockchip->vpcie1v8)) {
1019 err = regulator_enable(rockchip->vpcie1v8);
1020 if (err) {
1021 dev_err(dev, "fail to enable vpcie1v8 regulator\n");
1022 goto err_disable_3v3;
1023 }
1024 }
1025
1026 if (!IS_ERR(rockchip->vpcie0v9)) {
1027 err = regulator_enable(rockchip->vpcie0v9);
1028 if (err) {
1029 dev_err(dev, "fail to enable vpcie0v9 regulator\n");
1030 goto err_disable_1v8;
1031 }
1032 }
1033
1034 return 0;
1035
1036err_disable_1v8:
1037 if (!IS_ERR(rockchip->vpcie1v8))
1038 regulator_disable(rockchip->vpcie1v8);
1039err_disable_3v3:
1040 if (!IS_ERR(rockchip->vpcie3v3))
1041 regulator_disable(rockchip->vpcie3v3);
1042err_out:
1043 return err;
1044}
1045
1046static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip)
1047{
1048 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) &
1049 (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK);
1050 rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT),
1051 PCIE_CORE_INT_MASK);
1052
1053 rockchip_pcie_enable_bw_int(rockchip);
1054}
1055
1056static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
1057 irq_hw_number_t hwirq)
1058{
1059 irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
1060 irq_set_chip_data(irq, domain->host_data);
1061
1062 return 0;
1063}
1064
1065static const struct irq_domain_ops intx_domain_ops = {
1066 .map = rockchip_pcie_intx_map,
1067};
1068
1069static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
1070{
1071 struct device *dev = rockchip->dev;
1072 struct device_node *intc = of_get_next_child(dev->of_node, NULL);
1073
1074 if (!intc) {
1075 dev_err(dev, "missing child interrupt-controller node\n");
1076 return -EINVAL;
1077 }
1078
1079 rockchip->irq_domain = irq_domain_add_linear(intc, 4,
1080 &intx_domain_ops, rockchip);
1081 if (!rockchip->irq_domain) {
1082 dev_err(dev, "failed to get a INTx IRQ domain\n");
1083 return -EINVAL;
1084 }
1085
1086 return 0;
1087}
1088
1089static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip,
1090 int region_no, int type, u8 num_pass_bits,
1091 u32 lower_addr, u32 upper_addr)
1092{
1093 u32 ob_addr_0;
1094 u32 ob_addr_1;
1095 u32 ob_desc_0;
1096 u32 aw_offset;
1097
1098 if (region_no >= MAX_AXI_WRAPPER_REGION_NUM)
1099 return -EINVAL;
1100 if (num_pass_bits + 1 < 8)
1101 return -EINVAL;
1102 if (num_pass_bits > 63)
1103 return -EINVAL;
1104 if (region_no == 0) {
1105 if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits))
1106 return -EINVAL;
1107 }
1108 if (region_no != 0) {
1109 if (AXI_REGION_SIZE < (2ULL << num_pass_bits))
1110 return -EINVAL;
1111 }
1112
1113 aw_offset = (region_no << OB_REG_SIZE_SHIFT);
1114
1115 ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS;
1116 ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR;
1117 ob_addr_1 = upper_addr;
1118 ob_desc_0 = (1 << 23 | type);
1119
1120 rockchip_pcie_write(rockchip, ob_addr_0,
1121 PCIE_CORE_OB_REGION_ADDR0 + aw_offset);
1122 rockchip_pcie_write(rockchip, ob_addr_1,
1123 PCIE_CORE_OB_REGION_ADDR1 + aw_offset);
1124 rockchip_pcie_write(rockchip, ob_desc_0,
1125 PCIE_CORE_OB_REGION_DESC0 + aw_offset);
1126 rockchip_pcie_write(rockchip, 0,
1127 PCIE_CORE_OB_REGION_DESC1 + aw_offset);
1128
1129 return 0;
1130}
1131
1132static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip,
1133 int region_no, u8 num_pass_bits,
1134 u32 lower_addr, u32 upper_addr)
1135{
1136 u32 ib_addr_0;
1137 u32 ib_addr_1;
1138 u32 aw_offset;
1139
1140 if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM)
1141 return -EINVAL;
1142 if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED)
1143 return -EINVAL;
1144 if (num_pass_bits > 63)
1145 return -EINVAL;
1146
1147 aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT);
1148
1149 ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS;
1150 ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR;
1151 ib_addr_1 = upper_addr;
1152
1153 rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset);
1154 rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset);
1155
1156 return 0;
1157}
1158
1159static int rockchip_cfg_atu(struct rockchip_pcie *rockchip)
1160{
1161 struct device *dev = rockchip->dev;
1162 int offset;
1163 int err;
1164 int reg_no;
1165
1166 for (reg_no = 0; reg_no < (rockchip->mem_size >> 20); reg_no++) {
1167 err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1,
1168 AXI_WRAPPER_MEM_WRITE,
1169 20 - 1,
1170 rockchip->mem_bus_addr +
1171 (reg_no << 20),
1172 0);
1173 if (err) {
1174 dev_err(dev, "program RC mem outbound ATU failed\n");
1175 return err;
1176 }
1177 }
1178
1179 err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0);
1180 if (err) {
1181 dev_err(dev, "program RC mem inbound ATU failed\n");
1182 return err;
1183 }
1184
1185 offset = rockchip->mem_size >> 20;
1186 for (reg_no = 0; reg_no < (rockchip->io_size >> 20); reg_no++) {
1187 err = rockchip_pcie_prog_ob_atu(rockchip,
1188 reg_no + 1 + offset,
1189 AXI_WRAPPER_IO_WRITE,
1190 20 - 1,
1191 rockchip->io_bus_addr +
1192 (reg_no << 20),
1193 0);
1194 if (err) {
1195 dev_err(dev, "program RC io outbound ATU failed\n");
1196 return err;
1197 }
1198 }
1199
1200
1201 rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset,
1202 AXI_WRAPPER_NOR_MSG,
1203 20 - 1, 0, 0);
1204
1205 rockchip->msg_bus_addr = rockchip->mem_bus_addr +
1206 ((reg_no + offset) << 20);
1207 return err;
1208}
1209
1210static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip)
1211{
1212 u32 value;
1213 int err;
1214
1215
1216 writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF);
1217
1218
1219 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0,
1220 value, PCIE_LINK_IS_L2(value), 20,
1221 jiffies_to_usecs(5 * HZ));
1222 if (err) {
1223 dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n");
1224 return err;
1225 }
1226
1227 return 0;
1228}
1229
1230static int __maybe_unused rockchip_pcie_suspend_noirq(struct device *dev)
1231{
1232 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
1233 int ret;
1234
1235
1236 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) |
1237 PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK);
1238 rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK);
1239
1240 ret = rockchip_pcie_wait_l2(rockchip);
1241 if (ret) {
1242 rockchip_pcie_enable_interrupts(rockchip);
1243 return ret;
1244 }
1245
1246 phy_power_off(rockchip->phy);
1247 phy_exit(rockchip->phy);
1248
1249 clk_disable_unprepare(rockchip->clk_pcie_pm);
1250 clk_disable_unprepare(rockchip->hclk_pcie);
1251 clk_disable_unprepare(rockchip->aclk_perf_pcie);
1252 clk_disable_unprepare(rockchip->aclk_pcie);
1253
1254 return ret;
1255}
1256
1257static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev)
1258{
1259 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
1260 int err;
1261
1262 clk_prepare_enable(rockchip->clk_pcie_pm);
1263 clk_prepare_enable(rockchip->hclk_pcie);
1264 clk_prepare_enable(rockchip->aclk_perf_pcie);
1265 clk_prepare_enable(rockchip->aclk_pcie);
1266
1267 err = rockchip_pcie_init_port(rockchip);
1268 if (err)
1269 return err;
1270
1271 err = rockchip_cfg_atu(rockchip);
1272 if (err)
1273 return err;
1274
1275
1276 rockchip_pcie_update_txcredit_mui(rockchip);
1277 rockchip_pcie_enable_interrupts(rockchip);
1278
1279 return 0;
1280}
1281
1282static int rockchip_pcie_probe(struct platform_device *pdev)
1283{
1284 struct rockchip_pcie *rockchip;
1285 struct device *dev = &pdev->dev;
1286 struct pci_bus *bus, *child;
1287 struct resource_entry *win;
1288 resource_size_t io_base;
1289 struct resource *mem;
1290 struct resource *io;
1291 int err;
1292
1293 LIST_HEAD(res);
1294
1295 if (!dev->of_node)
1296 return -ENODEV;
1297
1298 rockchip = devm_kzalloc(dev, sizeof(*rockchip), GFP_KERNEL);
1299 if (!rockchip)
1300 return -ENOMEM;
1301
1302 platform_set_drvdata(pdev, rockchip);
1303 rockchip->dev = dev;
1304
1305 err = rockchip_pcie_parse_dt(rockchip);
1306 if (err)
1307 return err;
1308
1309 err = clk_prepare_enable(rockchip->aclk_pcie);
1310 if (err) {
1311 dev_err(dev, "unable to enable aclk_pcie clock\n");
1312 goto err_aclk_pcie;
1313 }
1314
1315 err = clk_prepare_enable(rockchip->aclk_perf_pcie);
1316 if (err) {
1317 dev_err(dev, "unable to enable aclk_perf_pcie clock\n");
1318 goto err_aclk_perf_pcie;
1319 }
1320
1321 err = clk_prepare_enable(rockchip->hclk_pcie);
1322 if (err) {
1323 dev_err(dev, "unable to enable hclk_pcie clock\n");
1324 goto err_hclk_pcie;
1325 }
1326
1327 err = clk_prepare_enable(rockchip->clk_pcie_pm);
1328 if (err) {
1329 dev_err(dev, "unable to enable hclk_pcie clock\n");
1330 goto err_pcie_pm;
1331 }
1332
1333 err = rockchip_pcie_set_vpcie(rockchip);
1334 if (err) {
1335 dev_err(dev, "failed to set vpcie regulator\n");
1336 goto err_set_vpcie;
1337 }
1338
1339 err = rockchip_pcie_init_port(rockchip);
1340 if (err)
1341 goto err_vpcie;
1342
1343 rockchip_pcie_enable_interrupts(rockchip);
1344
1345 err = rockchip_pcie_init_irq_domain(rockchip);
1346 if (err < 0)
1347 goto err_vpcie;
1348
1349 err = of_pci_get_host_bridge_resources(dev->of_node, 0, 0xff,
1350 &res, &io_base);
1351 if (err)
1352 goto err_vpcie;
1353
1354 err = devm_request_pci_bus_resources(dev, &res);
1355 if (err)
1356 goto err_free_res;
1357
1358
1359 resource_list_for_each_entry(win, &res) {
1360 switch (resource_type(win->res)) {
1361 case IORESOURCE_IO:
1362 io = win->res;
1363 io->name = "I/O";
1364 rockchip->io_size = resource_size(io);
1365 rockchip->io_bus_addr = io->start - win->offset;
1366 err = pci_remap_iospace(io, io_base);
1367 if (err) {
1368 dev_warn(dev, "error %d: failed to map resource %pR\n",
1369 err, io);
1370 continue;
1371 }
1372 rockchip->io = io;
1373 break;
1374 case IORESOURCE_MEM:
1375 mem = win->res;
1376 mem->name = "MEM";
1377 rockchip->mem_size = resource_size(mem);
1378 rockchip->mem_bus_addr = mem->start - win->offset;
1379 break;
1380 case IORESOURCE_BUS:
1381 rockchip->root_bus_nr = win->res->start;
1382 break;
1383 default:
1384 continue;
1385 }
1386 }
1387
1388 err = rockchip_cfg_atu(rockchip);
1389 if (err)
1390 goto err_free_res;
1391
1392 rockchip->msg_region = devm_ioremap(rockchip->dev,
1393 rockchip->msg_bus_addr, SZ_1M);
1394 if (!rockchip->msg_region) {
1395 err = -ENOMEM;
1396 goto err_free_res;
1397 }
1398
1399 bus = pci_scan_root_bus(&pdev->dev, 0, &rockchip_pcie_ops, rockchip, &res);
1400 if (!bus) {
1401 err = -ENOMEM;
1402 goto err_free_res;
1403 }
1404 rockchip->root_bus = bus;
1405
1406 pci_bus_size_bridges(bus);
1407 pci_bus_assign_resources(bus);
1408 list_for_each_entry(child, &bus->children, node)
1409 pcie_bus_configure_settings(child);
1410
1411 pci_bus_add_devices(bus);
1412 return 0;
1413
1414err_free_res:
1415 pci_free_resource_list(&res);
1416err_vpcie:
1417 if (!IS_ERR(rockchip->vpcie3v3))
1418 regulator_disable(rockchip->vpcie3v3);
1419 if (!IS_ERR(rockchip->vpcie1v8))
1420 regulator_disable(rockchip->vpcie1v8);
1421 if (!IS_ERR(rockchip->vpcie0v9))
1422 regulator_disable(rockchip->vpcie0v9);
1423err_set_vpcie:
1424 clk_disable_unprepare(rockchip->clk_pcie_pm);
1425err_pcie_pm:
1426 clk_disable_unprepare(rockchip->hclk_pcie);
1427err_hclk_pcie:
1428 clk_disable_unprepare(rockchip->aclk_perf_pcie);
1429err_aclk_perf_pcie:
1430 clk_disable_unprepare(rockchip->aclk_pcie);
1431err_aclk_pcie:
1432 return err;
1433}
1434
1435static int rockchip_pcie_remove(struct platform_device *pdev)
1436{
1437 struct device *dev = &pdev->dev;
1438 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
1439
1440 pci_stop_root_bus(rockchip->root_bus);
1441 pci_remove_root_bus(rockchip->root_bus);
1442 pci_unmap_iospace(rockchip->io);
1443 irq_domain_remove(rockchip->irq_domain);
1444
1445 phy_power_off(rockchip->phy);
1446 phy_exit(rockchip->phy);
1447
1448 clk_disable_unprepare(rockchip->clk_pcie_pm);
1449 clk_disable_unprepare(rockchip->hclk_pcie);
1450 clk_disable_unprepare(rockchip->aclk_perf_pcie);
1451 clk_disable_unprepare(rockchip->aclk_pcie);
1452
1453 if (!IS_ERR(rockchip->vpcie3v3))
1454 regulator_disable(rockchip->vpcie3v3);
1455 if (!IS_ERR(rockchip->vpcie1v8))
1456 regulator_disable(rockchip->vpcie1v8);
1457 if (!IS_ERR(rockchip->vpcie0v9))
1458 regulator_disable(rockchip->vpcie0v9);
1459
1460 return 0;
1461}
1462
1463static const struct dev_pm_ops rockchip_pcie_pm_ops = {
1464 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq,
1465 rockchip_pcie_resume_noirq)
1466};
1467
1468static const struct of_device_id rockchip_pcie_of_match[] = {
1469 { .compatible = "rockchip,rk3399-pcie", },
1470 {}
1471};
1472MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match);
1473
1474static struct platform_driver rockchip_pcie_driver = {
1475 .driver = {
1476 .name = "rockchip-pcie",
1477 .of_match_table = rockchip_pcie_of_match,
1478 .pm = &rockchip_pcie_pm_ops,
1479 },
1480 .probe = rockchip_pcie_probe,
1481 .remove = rockchip_pcie_remove,
1482};
1483module_platform_driver(rockchip_pcie_driver);
1484
1485MODULE_AUTHOR("Rockchip Inc");
1486MODULE_DESCRIPTION("Rockchip AXI PCIe driver");
1487MODULE_LICENSE("GPL v2");
1488
