087-0004-PCI-iproc-Add-iProc-PCIe-MSI-support.patch 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891
  1. From 3bc2b2348835f6edd33c383a2fbcf15fe3dac3b2 Mon Sep 17 00:00:00 2001
  2. From: Ray Jui <rjui@broadcom.com>
  3. Date: Wed, 6 Jan 2016 18:04:35 -0600
  4. Subject: [PATCH 4/5] PCI: iproc: Add iProc PCIe MSI support
  5. Add PCIe MSI support for both PAXB and PAXC interfaces on all iProc-based
  6. platforms.
  7. The iProc PCIe MSI support deploys an event queue-based implementation.
  8. Each event queue is serviced by a GIC interrupt and can support up to 64
  9. MSI vectors. Host memory is allocated for the event queues, and each event
  10. queue consists of 64 word-sized entries. MSI data is written to the lower
  11. 16-bit of each entry, whereas the upper 16-bit of the entry is reserved for
  12. the controller for internal processing.
  13. Each event queue is tracked by a head pointer and tail pointer. Head
  14. pointer indicates the next entry in the event queue to be processed by
  15. the driver and is updated by the driver after processing is done.
  16. The controller uses the tail pointer as the next MSI data insertion
  17. point. The controller ensures MSI data is flushed to host memory before
  18. updating the tail pointer and then triggering the interrupt.
  19. MSI IRQ affinity is supported by evenly distributing the interrupts to each
  20. CPU core. MSI vector is moved from one GIC interrupt to another in order
  21. to steer to the target CPU.
  22. Therefore, the actual number of supported MSI vectors is:
  23. M * 64 / N
  24. where M denotes the number of GIC interrupts (event queues), and N denotes
  25. the number of CPU cores.
  26. This iProc event queue-based MSI support should not be used with newer
  27. platforms with integrated MSI support in the GIC (e.g., giv2m or
  28. gicv3-its).
  29. [bhelgaas: fold in Kconfig fixes from Arnd Bergmann <arnd@arndb.de>]
  30. Signed-off-by: Ray Jui <rjui@broadcom.com>
  31. Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
  32. Reviewed-by: Anup Patel <anup.patel@broadcom.com>
  33. Reviewed-by: Vikram Prakash <vikramp@broadcom.com>
  34. Reviewed-by: Scott Branden <sbranden@broadcom.com>
  35. Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
  36. ---
  37. drivers/pci/host/Kconfig | 10 +
  38. drivers/pci/host/Makefile | 1 +
  39. drivers/pci/host/pcie-iproc-bcma.c | 1 +
  40. drivers/pci/host/pcie-iproc-msi.c | 675 +++++++++++++++++++++++++++++++++
  41. drivers/pci/host/pcie-iproc-platform.c | 1 +
  42. drivers/pci/host/pcie-iproc.c | 26 ++
  43. drivers/pci/host/pcie-iproc.h | 23 +-
  44. 7 files changed, 735 insertions(+), 2 deletions(-)
  45. create mode 100644 drivers/pci/host/pcie-iproc-msi.c
  46. --- a/drivers/pci/host/Kconfig
  47. +++ b/drivers/pci/host/Kconfig
  48. @@ -146,6 +146,16 @@ config PCIE_IPROC_BCMA
  49. Say Y here if you want to use the Broadcom iProc PCIe controller
  50. through the BCMA bus interface
  51. +config PCIE_IPROC_MSI
  52. + bool "Broadcom iProc PCIe MSI support"
  53. + depends on PCIE_IPROC_PLATFORM || PCIE_IPROC_BCMA
  54. + depends on PCI_MSI
  55. + select PCI_MSI_IRQ_DOMAIN
  56. + default ARCH_BCM_IPROC
  57. + help
  58. + Say Y here if you want to enable MSI support for Broadcom's iProc
  59. + PCIe controller
  60. +
  61. config PCIE_ALTERA
  62. bool "Altera PCIe controller"
  63. depends on ARM || NIOS2
  64. --- a/drivers/pci/host/Makefile
  65. +++ b/drivers/pci/host/Makefile
  66. @@ -15,6 +15,7 @@ obj-$(CONFIG_PCI_XGENE_MSI) += pci-xgene
  67. obj-$(CONFIG_PCI_LAYERSCAPE) += pci-layerscape.o
  68. obj-$(CONFIG_PCI_VERSATILE) += pci-versatile.o
  69. obj-$(CONFIG_PCIE_IPROC) += pcie-iproc.o
  70. +obj-$(CONFIG_PCIE_IPROC_MSI) += pcie-iproc-msi.o
  71. obj-$(CONFIG_PCIE_IPROC_PLATFORM) += pcie-iproc-platform.o
  72. obj-$(CONFIG_PCIE_IPROC_BCMA) += pcie-iproc-bcma.o
  73. obj-$(CONFIG_PCIE_ALTERA) += pcie-altera.o
  74. --- a/drivers/pci/host/pcie-iproc-bcma.c
  75. +++ b/drivers/pci/host/pcie-iproc-bcma.c
  76. @@ -55,6 +55,7 @@ static int iproc_pcie_bcma_probe(struct
  77. bcma_set_drvdata(bdev, pcie);
  78. pcie->base = bdev->io_addr;
  79. + pcie->base_addr = bdev->addr;
  80. res_mem.start = bdev->addr_s[0];
  81. res_mem.end = bdev->addr_s[0] + SZ_128M - 1;
  82. --- /dev/null
  83. +++ b/drivers/pci/host/pcie-iproc-msi.c
  84. @@ -0,0 +1,675 @@
  85. +/*
  86. + * Copyright (C) 2015 Broadcom Corporation
  87. + *
  88. + * This program is free software; you can redistribute it and/or
  89. + * modify it under the terms of the GNU General Public License as
  90. + * published by the Free Software Foundation version 2.
  91. + *
  92. + * This program is distributed "as is" WITHOUT ANY WARRANTY of any
  93. + * kind, whether express or implied; without even the implied warranty
  94. + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  95. + * GNU General Public License for more details.
  96. + */
  97. +
  98. +#include <linux/interrupt.h>
  99. +#include <linux/irqchip/chained_irq.h>
  100. +#include <linux/irqdomain.h>
  101. +#include <linux/msi.h>
  102. +#include <linux/of_irq.h>
  103. +#include <linux/of_pci.h>
  104. +#include <linux/pci.h>
  105. +
  106. +#include "pcie-iproc.h"
  107. +
  108. +#define IPROC_MSI_INTR_EN_SHIFT 11
  109. +#define IPROC_MSI_INTR_EN BIT(IPROC_MSI_INTR_EN_SHIFT)
  110. +#define IPROC_MSI_INT_N_EVENT_SHIFT 1
  111. +#define IPROC_MSI_INT_N_EVENT BIT(IPROC_MSI_INT_N_EVENT_SHIFT)
  112. +#define IPROC_MSI_EQ_EN_SHIFT 0
  113. +#define IPROC_MSI_EQ_EN BIT(IPROC_MSI_EQ_EN_SHIFT)
  114. +
  115. +#define IPROC_MSI_EQ_MASK 0x3f
  116. +
  117. +/* Max number of GIC interrupts */
  118. +#define NR_HW_IRQS 6
  119. +
  120. +/* Number of entries in each event queue */
  121. +#define EQ_LEN 64
  122. +
  123. +/* Size of each event queue memory region */
  124. +#define EQ_MEM_REGION_SIZE SZ_4K
  125. +
  126. +/* Size of each MSI address region */
  127. +#define MSI_MEM_REGION_SIZE SZ_4K
  128. +
  129. +enum iproc_msi_reg {
  130. + IPROC_MSI_EQ_PAGE = 0,
  131. + IPROC_MSI_EQ_PAGE_UPPER,
  132. + IPROC_MSI_PAGE,
  133. + IPROC_MSI_PAGE_UPPER,
  134. + IPROC_MSI_CTRL,
  135. + IPROC_MSI_EQ_HEAD,
  136. + IPROC_MSI_EQ_TAIL,
  137. + IPROC_MSI_INTS_EN,
  138. + IPROC_MSI_REG_SIZE,
  139. +};
  140. +
  141. +struct iproc_msi;
  142. +
  143. +/**
  144. + * iProc MSI group
  145. + *
  146. + * One MSI group is allocated per GIC interrupt, serviced by one iProc MSI
  147. + * event queue.
  148. + *
  149. + * @msi: pointer to iProc MSI data
  150. + * @gic_irq: GIC interrupt
  151. + * @eq: Event queue number
  152. + */
  153. +struct iproc_msi_grp {
  154. + struct iproc_msi *msi;
  155. + int gic_irq;
  156. + unsigned int eq;
  157. +};
  158. +
  159. +/**
  160. + * iProc event queue based MSI
  161. + *
  162. + * Only meant to be used on platforms without MSI support integrated into the
  163. + * GIC.
  164. + *
  165. + * @pcie: pointer to iProc PCIe data
  166. + * @reg_offsets: MSI register offsets
  167. + * @grps: MSI groups
  168. + * @nr_irqs: number of total interrupts connected to GIC
  169. + * @nr_cpus: number of toal CPUs
  170. + * @has_inten_reg: indicates the MSI interrupt enable register needs to be
  171. + * set explicitly (required for some legacy platforms)
  172. + * @bitmap: MSI vector bitmap
  173. + * @bitmap_lock: lock to protect access to the MSI bitmap
  174. + * @nr_msi_vecs: total number of MSI vectors
  175. + * @inner_domain: inner IRQ domain
  176. + * @msi_domain: MSI IRQ domain
  177. + * @nr_eq_region: required number of 4K aligned memory region for MSI event
  178. + * queues
  179. + * @nr_msi_region: required number of 4K aligned address region for MSI posted
  180. + * writes
  181. + * @eq_cpu: pointer to allocated memory region for MSI event queues
  182. + * @eq_dma: DMA address of MSI event queues
  183. + * @msi_addr: MSI address
  184. + */
  185. +struct iproc_msi {
  186. + struct iproc_pcie *pcie;
  187. + const u16 (*reg_offsets)[IPROC_MSI_REG_SIZE];
  188. + struct iproc_msi_grp *grps;
  189. + int nr_irqs;
  190. + int nr_cpus;
  191. + bool has_inten_reg;
  192. + unsigned long *bitmap;
  193. + struct mutex bitmap_lock;
  194. + unsigned int nr_msi_vecs;
  195. + struct irq_domain *inner_domain;
  196. + struct irq_domain *msi_domain;
  197. + unsigned int nr_eq_region;
  198. + unsigned int nr_msi_region;
  199. + void *eq_cpu;
  200. + dma_addr_t eq_dma;
  201. + phys_addr_t msi_addr;
  202. +};
  203. +
  204. +static const u16 iproc_msi_reg_paxb[NR_HW_IRQS][IPROC_MSI_REG_SIZE] = {
  205. + { 0x200, 0x2c0, 0x204, 0x2c4, 0x210, 0x250, 0x254, 0x208 },
  206. + { 0x200, 0x2c0, 0x204, 0x2c4, 0x214, 0x258, 0x25c, 0x208 },
  207. + { 0x200, 0x2c0, 0x204, 0x2c4, 0x218, 0x260, 0x264, 0x208 },
  208. + { 0x200, 0x2c0, 0x204, 0x2c4, 0x21c, 0x268, 0x26c, 0x208 },
  209. + { 0x200, 0x2c0, 0x204, 0x2c4, 0x220, 0x270, 0x274, 0x208 },
  210. + { 0x200, 0x2c0, 0x204, 0x2c4, 0x224, 0x278, 0x27c, 0x208 },
  211. +};
  212. +
  213. +static const u16 iproc_msi_reg_paxc[NR_HW_IRQS][IPROC_MSI_REG_SIZE] = {
  214. + { 0xc00, 0xc04, 0xc08, 0xc0c, 0xc40, 0xc50, 0xc60 },
  215. + { 0xc10, 0xc14, 0xc18, 0xc1c, 0xc44, 0xc54, 0xc64 },
  216. + { 0xc20, 0xc24, 0xc28, 0xc2c, 0xc48, 0xc58, 0xc68 },
  217. + { 0xc30, 0xc34, 0xc38, 0xc3c, 0xc4c, 0xc5c, 0xc6c },
  218. +};
  219. +
  220. +static inline u32 iproc_msi_read_reg(struct iproc_msi *msi,
  221. + enum iproc_msi_reg reg,
  222. + unsigned int eq)
  223. +{
  224. + struct iproc_pcie *pcie = msi->pcie;
  225. +
  226. + return readl_relaxed(pcie->base + msi->reg_offsets[eq][reg]);
  227. +}
  228. +
  229. +static inline void iproc_msi_write_reg(struct iproc_msi *msi,
  230. + enum iproc_msi_reg reg,
  231. + int eq, u32 val)
  232. +{
  233. + struct iproc_pcie *pcie = msi->pcie;
  234. +
  235. + writel_relaxed(val, pcie->base + msi->reg_offsets[eq][reg]);
  236. +}
  237. +
  238. +static inline u32 hwirq_to_group(struct iproc_msi *msi, unsigned long hwirq)
  239. +{
  240. + return (hwirq % msi->nr_irqs);
  241. +}
  242. +
  243. +static inline unsigned int iproc_msi_addr_offset(struct iproc_msi *msi,
  244. + unsigned long hwirq)
  245. +{
  246. + if (msi->nr_msi_region > 1)
  247. + return hwirq_to_group(msi, hwirq) * MSI_MEM_REGION_SIZE;
  248. + else
  249. + return hwirq_to_group(msi, hwirq) * sizeof(u32);
  250. +}
  251. +
  252. +static inline unsigned int iproc_msi_eq_offset(struct iproc_msi *msi, u32 eq)
  253. +{
  254. + if (msi->nr_eq_region > 1)
  255. + return eq * EQ_MEM_REGION_SIZE;
  256. + else
  257. + return eq * EQ_LEN * sizeof(u32);
  258. +}
  259. +
  260. +static struct irq_chip iproc_msi_irq_chip = {
  261. + .name = "iProc-MSI",
  262. +};
  263. +
  264. +static struct msi_domain_info iproc_msi_domain_info = {
  265. + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
  266. + MSI_FLAG_PCI_MSIX,
  267. + .chip = &iproc_msi_irq_chip,
  268. +};
  269. +
  270. +/*
  271. + * In iProc PCIe core, each MSI group is serviced by a GIC interrupt and a
  272. + * dedicated event queue. Each MSI group can support up to 64 MSI vectors.
  273. + *
  274. + * The number of MSI groups varies between different iProc SoCs. The total
  275. + * number of CPU cores also varies. To support MSI IRQ affinity, we
  276. + * distribute GIC interrupts across all available CPUs. MSI vector is moved
  277. + * from one GIC interrupt to another to steer to the target CPU.
  278. + *
  279. + * Assuming:
  280. + * - the number of MSI groups is M
  281. + * - the number of CPU cores is N
  282. + * - M is always a multiple of N
  283. + *
  284. + * Total number of raw MSI vectors = M * 64
  285. + * Total number of supported MSI vectors = (M * 64) / N
  286. + */
  287. +static inline int hwirq_to_cpu(struct iproc_msi *msi, unsigned long hwirq)
  288. +{
  289. + return (hwirq % msi->nr_cpus);
  290. +}
  291. +
  292. +static inline unsigned long hwirq_to_canonical_hwirq(struct iproc_msi *msi,
  293. + unsigned long hwirq)
  294. +{
  295. + return (hwirq - hwirq_to_cpu(msi, hwirq));
  296. +}
  297. +
  298. +static int iproc_msi_irq_set_affinity(struct irq_data *data,
  299. + const struct cpumask *mask, bool force)
  300. +{
  301. + struct iproc_msi *msi = irq_data_get_irq_chip_data(data);
  302. + int target_cpu = cpumask_first(mask);
  303. + int curr_cpu;
  304. +
  305. + curr_cpu = hwirq_to_cpu(msi, data->hwirq);
  306. + if (curr_cpu == target_cpu)
  307. + return IRQ_SET_MASK_OK_DONE;
  308. +
  309. + /* steer MSI to the target CPU */
  310. + data->hwirq = hwirq_to_canonical_hwirq(msi, data->hwirq) + target_cpu;
  311. +
  312. + return IRQ_SET_MASK_OK;
  313. +}
  314. +
  315. +static void iproc_msi_irq_compose_msi_msg(struct irq_data *data,
  316. + struct msi_msg *msg)
  317. +{
  318. + struct iproc_msi *msi = irq_data_get_irq_chip_data(data);
  319. + dma_addr_t addr;
  320. +
  321. + addr = msi->msi_addr + iproc_msi_addr_offset(msi, data->hwirq);
  322. + msg->address_lo = lower_32_bits(addr);
  323. + msg->address_hi = upper_32_bits(addr);
  324. + msg->data = data->hwirq;
  325. +}
  326. +
  327. +static struct irq_chip iproc_msi_bottom_irq_chip = {
  328. + .name = "MSI",
  329. + .irq_set_affinity = iproc_msi_irq_set_affinity,
  330. + .irq_compose_msi_msg = iproc_msi_irq_compose_msi_msg,
  331. +};
  332. +
  333. +static int iproc_msi_irq_domain_alloc(struct irq_domain *domain,
  334. + unsigned int virq, unsigned int nr_irqs,
  335. + void *args)
  336. +{
  337. + struct iproc_msi *msi = domain->host_data;
  338. + int hwirq;
  339. +
  340. + mutex_lock(&msi->bitmap_lock);
  341. +
  342. + /* Allocate 'nr_cpus' number of MSI vectors each time */
  343. + hwirq = bitmap_find_next_zero_area(msi->bitmap, msi->nr_msi_vecs, 0,
  344. + msi->nr_cpus, 0);
  345. + if (hwirq < msi->nr_msi_vecs) {
  346. + bitmap_set(msi->bitmap, hwirq, msi->nr_cpus);
  347. + } else {
  348. + mutex_unlock(&msi->bitmap_lock);
  349. + return -ENOSPC;
  350. + }
  351. +
  352. + mutex_unlock(&msi->bitmap_lock);
  353. +
  354. + irq_domain_set_info(domain, virq, hwirq, &iproc_msi_bottom_irq_chip,
  355. + domain->host_data, handle_simple_irq, NULL, NULL);
  356. +
  357. + return 0;
  358. +}
  359. +
  360. +static void iproc_msi_irq_domain_free(struct irq_domain *domain,
  361. + unsigned int virq, unsigned int nr_irqs)
  362. +{
  363. + struct irq_data *data = irq_domain_get_irq_data(domain, virq);
  364. + struct iproc_msi *msi = irq_data_get_irq_chip_data(data);
  365. + unsigned int hwirq;
  366. +
  367. + mutex_lock(&msi->bitmap_lock);
  368. +
  369. + hwirq = hwirq_to_canonical_hwirq(msi, data->hwirq);
  370. + bitmap_clear(msi->bitmap, hwirq, msi->nr_cpus);
  371. +
  372. + mutex_unlock(&msi->bitmap_lock);
  373. +
  374. + irq_domain_free_irqs_parent(domain, virq, nr_irqs);
  375. +}
  376. +
  377. +static const struct irq_domain_ops msi_domain_ops = {
  378. + .alloc = iproc_msi_irq_domain_alloc,
  379. + .free = iproc_msi_irq_domain_free,
  380. +};
  381. +
  382. +static inline u32 decode_msi_hwirq(struct iproc_msi *msi, u32 eq, u32 head)
  383. +{
  384. + u32 *msg, hwirq;
  385. + unsigned int offs;
  386. +
  387. + offs = iproc_msi_eq_offset(msi, eq) + head * sizeof(u32);
  388. + msg = (u32 *)(msi->eq_cpu + offs);
  389. + hwirq = *msg & IPROC_MSI_EQ_MASK;
  390. +
  391. + /*
  392. + * Since we have multiple hwirq mapped to a single MSI vector,
  393. + * now we need to derive the hwirq at CPU0. It can then be used to
  394. + * mapped back to virq.
  395. + */
  396. + return hwirq_to_canonical_hwirq(msi, hwirq);
  397. +}
  398. +
  399. +static void iproc_msi_handler(struct irq_desc *desc)
  400. +{
  401. + struct irq_chip *chip = irq_desc_get_chip(desc);
  402. + struct iproc_msi_grp *grp;
  403. + struct iproc_msi *msi;
  404. + struct iproc_pcie *pcie;
  405. + u32 eq, head, tail, nr_events;
  406. + unsigned long hwirq;
  407. + int virq;
  408. +
  409. + chained_irq_enter(chip, desc);
  410. +
  411. + grp = irq_desc_get_handler_data(desc);
  412. + msi = grp->msi;
  413. + pcie = msi->pcie;
  414. + eq = grp->eq;
  415. +
  416. + /*
  417. + * iProc MSI event queue is tracked by head and tail pointers. Head
  418. + * pointer indicates the next entry (MSI data) to be consumed by SW in
  419. + * the queue and needs to be updated by SW. iProc MSI core uses the
  420. + * tail pointer as the next data insertion point.
  421. + *
  422. + * Entries between head and tail pointers contain valid MSI data. MSI
  423. + * data is guaranteed to be in the event queue memory before the tail
  424. + * pointer is updated by the iProc MSI core.
  425. + */
  426. + head = iproc_msi_read_reg(msi, IPROC_MSI_EQ_HEAD,
  427. + eq) & IPROC_MSI_EQ_MASK;
  428. + do {
  429. + tail = iproc_msi_read_reg(msi, IPROC_MSI_EQ_TAIL,
  430. + eq) & IPROC_MSI_EQ_MASK;
  431. +
  432. + /*
  433. + * Figure out total number of events (MSI data) to be
  434. + * processed.
  435. + */
  436. + nr_events = (tail < head) ?
  437. + (EQ_LEN - (head - tail)) : (tail - head);
  438. + if (!nr_events)
  439. + break;
  440. +
  441. + /* process all outstanding events */
  442. + while (nr_events--) {
  443. + hwirq = decode_msi_hwirq(msi, eq, head);
  444. + virq = irq_find_mapping(msi->inner_domain, hwirq);
  445. + generic_handle_irq(virq);
  446. +
  447. + head++;
  448. + head %= EQ_LEN;
  449. + }
  450. +
  451. + /*
  452. + * Now all outstanding events have been processed. Update the
  453. + * head pointer.
  454. + */
  455. + iproc_msi_write_reg(msi, IPROC_MSI_EQ_HEAD, eq, head);
  456. +
  457. + /*
  458. + * Now go read the tail pointer again to see if there are new
  459. + * oustanding events that came in during the above window.
  460. + */
  461. + } while (true);
  462. +
  463. + chained_irq_exit(chip, desc);
  464. +}
  465. +
  466. +static void iproc_msi_enable(struct iproc_msi *msi)
  467. +{
  468. + int i, eq;
  469. + u32 val;
  470. +
  471. + /* Program memory region for each event queue */
  472. + for (i = 0; i < msi->nr_eq_region; i++) {
  473. + dma_addr_t addr = msi->eq_dma + (i * EQ_MEM_REGION_SIZE);
  474. +
  475. + iproc_msi_write_reg(msi, IPROC_MSI_EQ_PAGE, i,
  476. + lower_32_bits(addr));
  477. + iproc_msi_write_reg(msi, IPROC_MSI_EQ_PAGE_UPPER, i,
  478. + upper_32_bits(addr));
  479. + }
  480. +
  481. + /* Program address region for MSI posted writes */
  482. + for (i = 0; i < msi->nr_msi_region; i++) {
  483. + phys_addr_t addr = msi->msi_addr + (i * MSI_MEM_REGION_SIZE);
  484. +
  485. + iproc_msi_write_reg(msi, IPROC_MSI_PAGE, i,
  486. + lower_32_bits(addr));
  487. + iproc_msi_write_reg(msi, IPROC_MSI_PAGE_UPPER, i,
  488. + upper_32_bits(addr));
  489. + }
  490. +
  491. + for (eq = 0; eq < msi->nr_irqs; eq++) {
  492. + /* Enable MSI event queue */
  493. + val = IPROC_MSI_INTR_EN | IPROC_MSI_INT_N_EVENT |
  494. + IPROC_MSI_EQ_EN;
  495. + iproc_msi_write_reg(msi, IPROC_MSI_CTRL, eq, val);
  496. +
  497. + /*
  498. + * Some legacy platforms require the MSI interrupt enable
  499. + * register to be set explicitly.
  500. + */
  501. + if (msi->has_inten_reg) {
  502. + val = iproc_msi_read_reg(msi, IPROC_MSI_INTS_EN, eq);
  503. + val |= BIT(eq);
  504. + iproc_msi_write_reg(msi, IPROC_MSI_INTS_EN, eq, val);
  505. + }
  506. + }
  507. +}
  508. +
  509. +static void iproc_msi_disable(struct iproc_msi *msi)
  510. +{
  511. + u32 eq, val;
  512. +
  513. + for (eq = 0; eq < msi->nr_irqs; eq++) {
  514. + if (msi->has_inten_reg) {
  515. + val = iproc_msi_read_reg(msi, IPROC_MSI_INTS_EN, eq);
  516. + val &= ~BIT(eq);
  517. + iproc_msi_write_reg(msi, IPROC_MSI_INTS_EN, eq, val);
  518. + }
  519. +
  520. + val = iproc_msi_read_reg(msi, IPROC_MSI_CTRL, eq);
  521. + val &= ~(IPROC_MSI_INTR_EN | IPROC_MSI_INT_N_EVENT |
  522. + IPROC_MSI_EQ_EN);
  523. + iproc_msi_write_reg(msi, IPROC_MSI_CTRL, eq, val);
  524. + }
  525. +}
  526. +
  527. +static int iproc_msi_alloc_domains(struct device_node *node,
  528. + struct iproc_msi *msi)
  529. +{
  530. + msi->inner_domain = irq_domain_add_linear(NULL, msi->nr_msi_vecs,
  531. + &msi_domain_ops, msi);
  532. + if (!msi->inner_domain)
  533. + return -ENOMEM;
  534. +
  535. + msi->msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(node),
  536. + &iproc_msi_domain_info,
  537. + msi->inner_domain);
  538. + if (!msi->msi_domain) {
  539. + irq_domain_remove(msi->inner_domain);
  540. + return -ENOMEM;
  541. + }
  542. +
  543. + return 0;
  544. +}
  545. +
  546. +static void iproc_msi_free_domains(struct iproc_msi *msi)
  547. +{
  548. + if (msi->msi_domain)
  549. + irq_domain_remove(msi->msi_domain);
  550. +
  551. + if (msi->inner_domain)
  552. + irq_domain_remove(msi->inner_domain);
  553. +}
  554. +
  555. +static void iproc_msi_irq_free(struct iproc_msi *msi, unsigned int cpu)
  556. +{
  557. + int i;
  558. +
  559. + for (i = cpu; i < msi->nr_irqs; i += msi->nr_cpus) {
  560. + irq_set_chained_handler_and_data(msi->grps[i].gic_irq,
  561. + NULL, NULL);
  562. + }
  563. +}
  564. +
  565. +static int iproc_msi_irq_setup(struct iproc_msi *msi, unsigned int cpu)
  566. +{
  567. + int i, ret;
  568. + cpumask_var_t mask;
  569. + struct iproc_pcie *pcie = msi->pcie;
  570. +
  571. + for (i = cpu; i < msi->nr_irqs; i += msi->nr_cpus) {
  572. + irq_set_chained_handler_and_data(msi->grps[i].gic_irq,
  573. + iproc_msi_handler,
  574. + &msi->grps[i]);
  575. + /* Dedicate GIC interrupt to each CPU core */
  576. + if (alloc_cpumask_var(&mask, GFP_KERNEL)) {
  577. + cpumask_clear(mask);
  578. + cpumask_set_cpu(cpu, mask);
  579. + ret = irq_set_affinity(msi->grps[i].gic_irq, mask);
  580. + if (ret)
  581. + dev_err(pcie->dev,
  582. + "failed to set affinity for IRQ%d\n",
  583. + msi->grps[i].gic_irq);
  584. + free_cpumask_var(mask);
  585. + } else {
  586. + dev_err(pcie->dev, "failed to alloc CPU mask\n");
  587. + ret = -EINVAL;
  588. + }
  589. +
  590. + if (ret) {
  591. + /* Free all configured/unconfigured IRQs */
  592. + iproc_msi_irq_free(msi, cpu);
  593. + return ret;
  594. + }
  595. + }
  596. +
  597. + return 0;
  598. +}
  599. +
  600. +int iproc_msi_init(struct iproc_pcie *pcie, struct device_node *node)
  601. +{
  602. + struct iproc_msi *msi;
  603. + int i, ret;
  604. + unsigned int cpu;
  605. +
  606. + if (!of_device_is_compatible(node, "brcm,iproc-msi"))
  607. + return -ENODEV;
  608. +
  609. + if (!of_find_property(node, "msi-controller", NULL))
  610. + return -ENODEV;
  611. +
  612. + if (pcie->msi)
  613. + return -EBUSY;
  614. +
  615. + msi = devm_kzalloc(pcie->dev, sizeof(*msi), GFP_KERNEL);
  616. + if (!msi)
  617. + return -ENOMEM;
  618. +
  619. + msi->pcie = pcie;
  620. + pcie->msi = msi;
  621. + msi->msi_addr = pcie->base_addr;
  622. + mutex_init(&msi->bitmap_lock);
  623. + msi->nr_cpus = num_possible_cpus();
  624. +
  625. + msi->nr_irqs = of_irq_count(node);
  626. + if (!msi->nr_irqs) {
  627. + dev_err(pcie->dev, "found no MSI GIC interrupt\n");
  628. + return -ENODEV;
  629. + }
  630. +
  631. + if (msi->nr_irqs > NR_HW_IRQS) {
  632. + dev_warn(pcie->dev, "too many MSI GIC interrupts defined %d\n",
  633. + msi->nr_irqs);
  634. + msi->nr_irqs = NR_HW_IRQS;
  635. + }
  636. +
  637. + if (msi->nr_irqs < msi->nr_cpus) {
  638. + dev_err(pcie->dev,
  639. + "not enough GIC interrupts for MSI affinity\n");
  640. + return -EINVAL;
  641. + }
  642. +
  643. + if (msi->nr_irqs % msi->nr_cpus != 0) {
  644. + msi->nr_irqs -= msi->nr_irqs % msi->nr_cpus;
  645. + dev_warn(pcie->dev, "Reducing number of interrupts to %d\n",
  646. + msi->nr_irqs);
  647. + }
  648. +
  649. + switch (pcie->type) {
  650. + case IPROC_PCIE_PAXB:
  651. + msi->reg_offsets = iproc_msi_reg_paxb;
  652. + msi->nr_eq_region = 1;
  653. + msi->nr_msi_region = 1;
  654. + break;
  655. + case IPROC_PCIE_PAXC:
  656. + msi->reg_offsets = iproc_msi_reg_paxc;
  657. + msi->nr_eq_region = msi->nr_irqs;
  658. + msi->nr_msi_region = msi->nr_irqs;
  659. + break;
  660. + default:
  661. + dev_err(pcie->dev, "incompatible iProc PCIe interface\n");
  662. + return -EINVAL;
  663. + }
  664. +
  665. + if (of_find_property(node, "brcm,pcie-msi-inten", NULL))
  666. + msi->has_inten_reg = true;
  667. +
  668. + msi->nr_msi_vecs = msi->nr_irqs * EQ_LEN;
  669. + msi->bitmap = devm_kcalloc(pcie->dev, BITS_TO_LONGS(msi->nr_msi_vecs),
  670. + sizeof(*msi->bitmap), GFP_KERNEL);
  671. + if (!msi->bitmap)
  672. + return -ENOMEM;
  673. +
  674. + msi->grps = devm_kcalloc(pcie->dev, msi->nr_irqs, sizeof(*msi->grps),
  675. + GFP_KERNEL);
  676. + if (!msi->grps)
  677. + return -ENOMEM;
  678. +
  679. + for (i = 0; i < msi->nr_irqs; i++) {
  680. + unsigned int irq = irq_of_parse_and_map(node, i);
  681. +
  682. + if (!irq) {
  683. + dev_err(pcie->dev, "unable to parse/map interrupt\n");
  684. + ret = -ENODEV;
  685. + goto free_irqs;
  686. + }
  687. + msi->grps[i].gic_irq = irq;
  688. + msi->grps[i].msi = msi;
  689. + msi->grps[i].eq = i;
  690. + }
  691. +
  692. + /* Reserve memory for event queue and make sure memories are zeroed */
  693. + msi->eq_cpu = dma_zalloc_coherent(pcie->dev,
  694. + msi->nr_eq_region * EQ_MEM_REGION_SIZE,
  695. + &msi->eq_dma, GFP_KERNEL);
  696. + if (!msi->eq_cpu) {
  697. + ret = -ENOMEM;
  698. + goto free_irqs;
  699. + }
  700. +
  701. + ret = iproc_msi_alloc_domains(node, msi);
  702. + if (ret) {
  703. + dev_err(pcie->dev, "failed to create MSI domains\n");
  704. + goto free_eq_dma;
  705. + }
  706. +
  707. + for_each_online_cpu(cpu) {
  708. + ret = iproc_msi_irq_setup(msi, cpu);
  709. + if (ret)
  710. + goto free_msi_irq;
  711. + }
  712. +
  713. + iproc_msi_enable(msi);
  714. +
  715. + return 0;
  716. +
  717. +free_msi_irq:
  718. + for_each_online_cpu(cpu)
  719. + iproc_msi_irq_free(msi, cpu);
  720. + iproc_msi_free_domains(msi);
  721. +
  722. +free_eq_dma:
  723. + dma_free_coherent(pcie->dev, msi->nr_eq_region * EQ_MEM_REGION_SIZE,
  724. + msi->eq_cpu, msi->eq_dma);
  725. +
  726. +free_irqs:
  727. + for (i = 0; i < msi->nr_irqs; i++) {
  728. + if (msi->grps[i].gic_irq)
  729. + irq_dispose_mapping(msi->grps[i].gic_irq);
  730. + }
  731. + pcie->msi = NULL;
  732. + return ret;
  733. +}
  734. +EXPORT_SYMBOL(iproc_msi_init);
  735. +
  736. +void iproc_msi_exit(struct iproc_pcie *pcie)
  737. +{
  738. + struct iproc_msi *msi = pcie->msi;
  739. + unsigned int i, cpu;
  740. +
  741. + if (!msi)
  742. + return;
  743. +
  744. + iproc_msi_disable(msi);
  745. +
  746. + for_each_online_cpu(cpu)
  747. + iproc_msi_irq_free(msi, cpu);
  748. +
  749. + iproc_msi_free_domains(msi);
  750. +
  751. + dma_free_coherent(pcie->dev, msi->nr_eq_region * EQ_MEM_REGION_SIZE,
  752. + msi->eq_cpu, msi->eq_dma);
  753. +
  754. + for (i = 0; i < msi->nr_irqs; i++) {
  755. + if (msi->grps[i].gic_irq)
  756. + irq_dispose_mapping(msi->grps[i].gic_irq);
  757. + }
  758. +}
  759. +EXPORT_SYMBOL(iproc_msi_exit);
  760. --- a/drivers/pci/host/pcie-iproc-platform.c
  761. +++ b/drivers/pci/host/pcie-iproc-platform.c
  762. @@ -71,6 +71,7 @@ static int iproc_pcie_pltfm_probe(struct
  763. dev_err(pcie->dev, "unable to map controller registers\n");
  764. return -ENOMEM;
  765. }
  766. + pcie->base_addr = reg.start;
  767. if (of_property_read_bool(np, "brcm,pcie-ob")) {
  768. u32 val;
  769. --- a/drivers/pci/host/pcie-iproc.c
  770. +++ b/drivers/pci/host/pcie-iproc.c
  771. @@ -440,6 +440,26 @@ static int iproc_pcie_map_ranges(struct
  772. return 0;
  773. }
  774. +static int iproc_pcie_msi_enable(struct iproc_pcie *pcie)
  775. +{
  776. + struct device_node *msi_node;
  777. +
  778. + msi_node = of_parse_phandle(pcie->dev->of_node, "msi-parent", 0);
  779. + if (!msi_node)
  780. + return -ENODEV;
  781. +
  782. + /*
  783. + * If another MSI controller is being used, the call below should fail
  784. + * but that is okay
  785. + */
  786. + return iproc_msi_init(pcie, msi_node);
  787. +}
  788. +
  789. +static void iproc_pcie_msi_disable(struct iproc_pcie *pcie)
  790. +{
  791. + iproc_msi_exit(pcie);
  792. +}
  793. +
  794. int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)
  795. {
  796. int ret;
  797. @@ -507,6 +527,10 @@ int iproc_pcie_setup(struct iproc_pcie *
  798. iproc_pcie_enable(pcie);
  799. + if (IS_ENABLED(CONFIG_PCI_MSI))
  800. + if (iproc_pcie_msi_enable(pcie))
  801. + dev_info(pcie->dev, "not using iProc MSI\n");
  802. +
  803. pci_scan_child_bus(bus);
  804. pci_assign_unassigned_bus_resources(bus);
  805. pci_fixup_irqs(pci_common_swizzle, pcie->map_irq);
  806. @@ -531,6 +555,8 @@ int iproc_pcie_remove(struct iproc_pcie
  807. pci_stop_root_bus(pcie->root_bus);
  808. pci_remove_root_bus(pcie->root_bus);
  809. + iproc_pcie_msi_disable(pcie);
  810. +
  811. phy_power_off(pcie->phy);
  812. phy_exit(pcie->phy);
  813. --- a/drivers/pci/host/pcie-iproc.h
  814. +++ b/drivers/pci/host/pcie-iproc.h
  815. @@ -41,6 +41,8 @@ struct iproc_pcie_ob {
  816. resource_size_t window_size;
  817. };
  818. +struct iproc_msi;
  819. +
  820. /**
  821. * iProc PCIe device
  822. *
  823. @@ -48,19 +50,21 @@ struct iproc_pcie_ob {
  824. * @type: iProc PCIe interface type
  825. * @reg_offsets: register offsets
  826. * @base: PCIe host controller I/O register base
  827. + * @base_addr: PCIe host controller register base physical address
  828. * @sysdata: Per PCI controller data (ARM-specific)
  829. * @root_bus: pointer to root bus
  830. * @phy: optional PHY device that controls the Serdes
  831. - * @irqs: interrupt IDs
  832. * @map_irq: function callback to map interrupts
  833. - * @need_ob_cfg: indidates SW needs to configure the outbound mapping window
  834. + * @need_ob_cfg: indicates SW needs to configure the outbound mapping window
  835. * @ob: outbound mapping parameters
  836. + * @msi: MSI data
  837. */
  838. struct iproc_pcie {
  839. struct device *dev;
  840. enum iproc_pcie_type type;
  841. const u16 *reg_offsets;
  842. void __iomem *base;
  843. + phys_addr_t base_addr;
  844. #ifdef CONFIG_ARM
  845. struct pci_sys_data sysdata;
  846. #endif
  847. @@ -69,9 +73,24 @@ struct iproc_pcie {
  848. int (*map_irq)(const struct pci_dev *, u8, u8);
  849. bool need_ob_cfg;
  850. struct iproc_pcie_ob ob;
  851. + struct iproc_msi *msi;
  852. };
  853. int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res);
  854. int iproc_pcie_remove(struct iproc_pcie *pcie);
  855. +#ifdef CONFIG_PCIE_IPROC_MSI
  856. +int iproc_msi_init(struct iproc_pcie *pcie, struct device_node *node);
  857. +void iproc_msi_exit(struct iproc_pcie *pcie);
  858. +#else
  859. +static inline int iproc_msi_init(struct iproc_pcie *pcie,
  860. + struct device_node *node)
  861. +{
  862. + return -ENODEV;
  863. +}
  864. +static inline void iproc_msi_exit(struct iproc_pcie *pcie)
  865. +{
  866. +}
  867. +#endif
  868. +
  869. #endif /* _PCIE_IPROC_H */