first and last commit
This commit is contained in:
Chris Nutter committed
commit
1bf4efd552
2612 files changed
+636500
No files matched your search
@@ -0,0 +1,752 @@
|
||||
/*
|
||||
* Copyright (C) 2013 Apple Inc. All rights reserved.
|
||||
*
|
||||
* This document is the property of Apple Inc.
|
||||
* It is considered confidential and proprietary.
|
||||
*
|
||||
* This document may not be reproduced or transmitted in any form,
|
||||
* in whole or in part, without the express written permission of
|
||||
* Apple Inc.
|
||||
*/
|
||||
#include <debug.h>
|
||||
#include <drivers/dart.h>
|
||||
#include <drivers/thunderbolt/nhi.h>
|
||||
#include <drivers/pci.h>
|
||||
#include <platform.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/task.h>
|
||||
#include <sys.h>
|
||||
#include "nhi_protocol.h"
|
||||
|
||||
#define NHID_INIT (DEBUG_INFO)
|
||||
// These will lead to deadlock if thunderbolt is used for the serial console
|
||||
#define NHID_REG (DEBUG_SPEW)
|
||||
#define NHID_DMA (DEBUG_SPEW)
|
||||
#define NHID_RXPACKET_DUMP (DEBUG_NEVER)
|
||||
#define NHID_TXPACKET_DUMP (DEBUG_NEVER)
|
||||
|
||||
#define NHI_PROBE_MAX_BRIDGES (8)
|
||||
#define NHI_PROBE_MIN_BAR_SIZE (0x40000)
|
||||
|
||||
#define NHI_MAX_RINGS (1)
|
||||
|
||||
#define NHI_BOOKKEEPING_IOVM_BASE (0x80100000)
|
||||
#define NHI_BOOKKEEPING_PAGES (4)
|
||||
#define NHI_BOOKKEEPING_SIZE (NHI_BOOKKEEPING_PAGES * DART_PAGE_SIZE)
|
||||
#define NHI_BUFFER_IOVM_BASE (0x80200000)
|
||||
#if NHI_BOOKKEEPING_IOVM_BASE + NHI_BOOKKEEPING_SIZE > NHI_BUFFER_IOVM_BASE
|
||||
#error "NHI bookkeeping and buffer IOVM ranges overlap"
|
||||
#endif
|
||||
|
||||
typedef struct nhi_ring {
|
||||
uint64_t buffers_iovm_addr;
|
||||
uint64_t descriptors_iovm_addr;
|
||||
uint32_t num_buffers;
|
||||
uint32_t buffer_size;
|
||||
uint32_t next_buffer_index;
|
||||
|
||||
// All data structures after this comment are mapped via the
|
||||
// DART, so software MUST NOT assume they will stay constant.
|
||||
// Values must be copied to a safe location and then validated
|
||||
void *buffers;
|
||||
void *descriptors;
|
||||
} nhi_ring_t;
|
||||
|
||||
struct nhi {
|
||||
nhi_ring_t tx_rings[NHI_MAX_RINGS];
|
||||
nhi_ring_t rx_rings[NHI_MAX_RINGS];
|
||||
|
||||
uintptr_t bookkeeping_base;
|
||||
size_t bookkeeping_size;
|
||||
size_t bookkeeping_used;
|
||||
|
||||
size_t buffer_space_mapped;
|
||||
|
||||
pci_device_t pci_device;
|
||||
uintptr_t bar0;
|
||||
uint32_t dart_id;
|
||||
pci_device_t upstream_bridge;
|
||||
};
|
||||
|
||||
static void nhi_quiesce_pci(nhi_t *nhi);
|
||||
|
||||
static void nhi_write_reg(nhi_t *nhi, uint32_t offset, uint32_t value)
|
||||
{
|
||||
dprintf(NHID_REG, "nhi: write register 0x%05x (0x%llx) <- 0x%08x\n", offset, nhi->bar0 + offset, value);
|
||||
*(volatile uint32_t *)(nhi->bar0 + offset) = value;
|
||||
}
|
||||
|
||||
static uint32_t nhi_read_reg(nhi_t *nhi, uint32_t offset)
|
||||
{
|
||||
uint32_t value;
|
||||
value = *(volatile uint32_t *)(nhi->bar0 + offset);
|
||||
dprintf(NHID_REG, "nhi: read register 0x%05x (0x%llx) -> 0x%08x\n", offset, nhi->bar0 + offset, value);
|
||||
return value;
|
||||
}
|
||||
|
||||
static nhi_rx_buffer_desc_t *nhi_get_rx_buffer_desc(nhi_t *nhi, uint32_t ring_idx, uint32_t buffer_idx)
|
||||
{
|
||||
nhi_ring_t *ring;
|
||||
|
||||
ASSERT(nhi != NULL);
|
||||
ASSERT(ring_idx < NHI_MAX_RINGS);
|
||||
|
||||
ring = &nhi->rx_rings[ring_idx];
|
||||
ASSERT(buffer_idx < ring->num_buffers);
|
||||
|
||||
return (nhi_rx_buffer_desc_t *)ring->descriptors + buffer_idx;
|
||||
}
|
||||
|
||||
static nhi_tx_buffer_desc_t *nhi_get_tx_buffer_desc(nhi_t *nhi, uint32_t ring_idx, uint32_t buffer_idx)
|
||||
{
|
||||
nhi_ring_t *ring;
|
||||
|
||||
ASSERT(nhi != NULL);
|
||||
ASSERT(ring_idx < NHI_MAX_RINGS);
|
||||
|
||||
ring = &nhi->tx_rings[ring_idx];
|
||||
ASSERT(buffer_idx < ring->num_buffers);
|
||||
|
||||
return (nhi_tx_buffer_desc_t *)ring->descriptors + buffer_idx;
|
||||
}
|
||||
|
||||
static void *nhi_ring_get_buffer(nhi_ring_t *ring, uint32_t buffer_idx)
|
||||
{
|
||||
ASSERT(ring != NULL);
|
||||
ASSERT(buffer_idx < ring->num_buffers);
|
||||
|
||||
return (void *)((uint8_t *)ring->buffers + buffer_idx * ring->buffer_size);
|
||||
}
|
||||
|
||||
static uint64_t nhi_ring_get_buffer_iovm_addr(nhi_ring_t *ring, uint32_t buffer_idx)
|
||||
{
|
||||
ASSERT(ring != NULL);
|
||||
ASSERT(buffer_idx < ring->num_buffers);
|
||||
|
||||
return ring->buffers_iovm_addr + buffer_idx * ring->buffer_size;
|
||||
}
|
||||
|
||||
void nhi_debug_write(pci_device_t bridge, uint32_t vcap, uint8_t space, uint8_t port, uint16_t offset, uint32_t value)
|
||||
{
|
||||
uint32_t cmd;
|
||||
|
||||
cmd = offset | (port << 13) | (space << 19) | (1 << 21) | (0 << 22);
|
||||
|
||||
pci_config_write32(bridge, vcap + 0x34, value);
|
||||
pci_config_write32(bridge, vcap + 0x30, cmd | (1 << 30));
|
||||
spin(20);
|
||||
|
||||
for (int retries = 5; retries > 0; retries--) {
|
||||
do {
|
||||
cmd = pci_config_read32(bridge, vcap + 0x30);
|
||||
if ((cmd & (1U << 30)) == 0)
|
||||
break;
|
||||
if (cmd == 0xffffffff)
|
||||
return;
|
||||
spin(100);
|
||||
} while(1);
|
||||
|
||||
if (cmd & (1U << 31)) {
|
||||
spin(2000);
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t nhi_debug_read(pci_device_t bridge, uint32_t vcap, uint8_t space, uint8_t port, uint16_t offset)
|
||||
{
|
||||
uint32_t cmd;
|
||||
|
||||
cmd = offset | (port << 13) | (space << 19) | (0 << 22);
|
||||
|
||||
pci_config_write32(bridge, vcap + 0x30, cmd | (1 << 30));
|
||||
spin(20);
|
||||
|
||||
for (int retries = 5; retries > 0; retries--) {
|
||||
do {
|
||||
cmd = pci_config_read32(bridge, vcap + 0x30);
|
||||
if ((cmd & (1U << 30)) == 0)
|
||||
break;
|
||||
if (cmd == 0xffffffff)
|
||||
return 0xcdcdcdcd;
|
||||
spin(100);
|
||||
} while(1);
|
||||
|
||||
if (cmd & (1U << 31)) {
|
||||
spin(2000);
|
||||
continue;
|
||||
}
|
||||
return pci_config_read32(bridge, vcap + 0x38);
|
||||
}
|
||||
|
||||
return 0xabababab;
|
||||
}
|
||||
|
||||
static void debug_dump_packet(uint8_t *buffer, uint32_t bytes)
|
||||
{
|
||||
for (uint32_t off = 0; off < bytes; off += 4) {
|
||||
uint32_t dword;
|
||||
if (off % 16 == 0)
|
||||
printf("[0x%03x] ", off);
|
||||
memcpy(&dword, buffer + off, 4);
|
||||
printf("%08x ", ntohl(dword));
|
||||
if (off % 16 == 12)
|
||||
printf("\n");
|
||||
}
|
||||
if ((bytes) % 16 != 0)
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
void nhi_dump_config_space(pci_device_t bridge)
|
||||
{
|
||||
uint32_t offset;
|
||||
|
||||
offset = pci_find_extended_capability(bridge, 0xb, 0);
|
||||
if (offset == 0) {
|
||||
printf("couldn't get vendor-specific ecap\n");
|
||||
return;
|
||||
}
|
||||
|
||||
#if 0
|
||||
for (int j = 0; j < 0; j++) {
|
||||
printf("Port %d port configuration space\n", j);
|
||||
for (uint32_t i = 0; i < 8; i += 1) {
|
||||
uint32_t value = nhi_debug_read(bridge, offset, 1, j, i);
|
||||
spin(1000);
|
||||
if (i % 4 == 0)
|
||||
printf("%02x ", i);
|
||||
printf(" %08x", value);
|
||||
if (i % 4 == 3)
|
||||
printf("\n");
|
||||
}
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
for (int j = 0; j < 6; j++) {
|
||||
printf("Port %d path configuration space\n", j);
|
||||
for (uint32_t i = 0; i < 16; i += 1) {
|
||||
uint32_t value = nhi_debug_read(bridge, offset, 0, j, i);
|
||||
spin(1000);
|
||||
if (i % 4 == 0)
|
||||
printf("%02x ", i);
|
||||
printf(" %08x", value);
|
||||
if (i % 4 == 3)
|
||||
printf("\n");
|
||||
}
|
||||
}
|
||||
printf("\n");
|
||||
#endif
|
||||
|
||||
printf("Device configuration space\n");
|
||||
for (int i = 0; i < 5; i += 1) {
|
||||
uint32_t value = nhi_debug_read(bridge, offset, 2, 0, i);
|
||||
spin(1000);
|
||||
if (i % 4 == 0)
|
||||
printf("%02x ", i);
|
||||
printf(" %08x", value);
|
||||
if (i % 4 == 3)
|
||||
printf("\n");
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
nhi_t *nhi_init(pci_device_t bridge, uint32_t dart_id)
|
||||
{
|
||||
nhi_t *nhi = NULL;
|
||||
uint32_t class_code;
|
||||
uint32_t vendor_id;
|
||||
uintptr_t bar0;
|
||||
pci_device_t dev;
|
||||
pci_device_t temp_bridge;
|
||||
pci_device_t probed_bridges[NHI_PROBE_MAX_BRIDGES] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL };
|
||||
|
||||
ASSERT(bridge != NULL);
|
||||
|
||||
temp_bridge = bridge;
|
||||
for (unsigned i = 0; i < NHI_PROBE_MAX_BRIDGES; i++) {
|
||||
dev = pci_bridge_probe(temp_bridge, 0, 50000);
|
||||
if (dev == NULL)
|
||||
break;
|
||||
|
||||
if (dev == temp_bridge) {
|
||||
// We got the bridge back. Usually a sign of misconfiguration
|
||||
// in our PCI stack
|
||||
dprintf(DEBUG_CRITICAL, "nhi: probe from bridge %s returned itself\n", pci_get_name(temp_bridge));
|
||||
dev = NULL;
|
||||
break;
|
||||
}
|
||||
|
||||
if (pci_is_bridge(dev)) {
|
||||
dprintf(NHID_INIT, "nhi: probe found bridge: %s\n", pci_get_name(dev));
|
||||
pci_bridge_assign_secondary_bus(dev);
|
||||
probed_bridges[i] = dev;
|
||||
temp_bridge = dev;
|
||||
dev = NULL;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (dev == NULL) {
|
||||
dprintf(DEBUG_CRITICAL, "nhi: couldn't find a PCI device\n");
|
||||
goto error;
|
||||
}
|
||||
|
||||
dprintf(NHID_INIT, "nhi: found NHI PCI device %s\n", pci_get_name(dev));
|
||||
|
||||
class_code = pci_get_class_code(dev);
|
||||
vendor_id = pci_get_vendor_id(dev);
|
||||
|
||||
if (class_code != NHI_CLASS_CODE) {
|
||||
dprintf(DEBUG_CRITICAL, "nhi: incorrect class code 0x%x\n", class_code);
|
||||
goto error;
|
||||
}
|
||||
if (vendor_id != NHI_VENDOR_ID) {
|
||||
dprintf(DEBUG_CRITICAL, "nhi: incorrect vendor ID 0x%x\n", vendor_id);
|
||||
goto error;
|
||||
}
|
||||
|
||||
if (pci_get_bar_size(dev, 0) < NHI_PROBE_MIN_BAR_SIZE) {
|
||||
dprintf(DEBUG_CRITICAL, "nhi: BAR0 size 0x%llx too small\n", pci_get_bar_size(dev, 0));
|
||||
goto error;
|
||||
}
|
||||
bar0 = pci_map_bar(dev, 0);
|
||||
if (bar0 == 0) {
|
||||
dprintf(DEBUG_CRITICAL, "nhi: couldn't map BAR0\n");
|
||||
goto error;
|
||||
}
|
||||
|
||||
dprintf(NHID_INIT, "nhi: BAR0 mapped at 0x%llx\n", bar0);
|
||||
|
||||
nhi = calloc(sizeof(*nhi), 1);
|
||||
nhi->pci_device = dev;
|
||||
nhi->bar0 = bar0;
|
||||
nhi->dart_id = dart_id;
|
||||
|
||||
nhi->bookkeeping_size = NHI_BOOKKEEPING_PAGES * DART_PAGE_SIZE;
|
||||
nhi->bookkeeping_base = (uintptr_t)memalign(nhi->bookkeeping_size, DART_PAGE_SIZE);
|
||||
dart_map_page_range(nhi->dart_id, nhi->bookkeeping_base, NHI_BOOKKEEPING_IOVM_BASE, NHI_BOOKKEEPING_PAGES, false);
|
||||
|
||||
pci_bus_master_enable(dev, true);
|
||||
pci_memory_space_enable(dev, true);
|
||||
|
||||
// set bus master and memory space enable on all the bridges our probe found
|
||||
for (unsigned i = 0; i < NHI_PROBE_MAX_BRIDGES && probed_bridges[i] != NULL; i++) {
|
||||
pci_bus_master_enable(probed_bridges[i], true);
|
||||
pci_memory_space_enable(probed_bridges[i], true);
|
||||
}
|
||||
nhi->upstream_bridge = probed_bridges[0];
|
||||
|
||||
return nhi;
|
||||
|
||||
error:
|
||||
dprintf(DEBUG_CRITICAL, "nhi: initialization failed\n");
|
||||
if (nhi != NULL) {
|
||||
if (nhi->pci_device != NULL)
|
||||
nhi_quiesce_pci(nhi);
|
||||
if (nhi->bookkeeping_base != 0) {
|
||||
dart_unmap_page_range(nhi->dart_id, NHI_BOOKKEEPING_IOVM_BASE, NHI_BOOKKEEPING_PAGES);
|
||||
free((void *)nhi->bookkeeping_base);
|
||||
nhi->bookkeeping_base = 0;
|
||||
}
|
||||
|
||||
free(nhi);
|
||||
} else {
|
||||
for (unsigned i = 0; i < NHI_PROBE_MAX_BRIDGES; i++) {
|
||||
if (probed_bridges[NHI_PROBE_MAX_BRIDGES - i - 1] == NULL)
|
||||
continue;
|
||||
pci_free(probed_bridges[NHI_PROBE_MAX_BRIDGES - i - 1]);
|
||||
}
|
||||
if (dev != NULL)
|
||||
pci_free(dev);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void nhi_quiesce_pci(nhi_t *nhi)
|
||||
{
|
||||
pci_device_t device;
|
||||
pci_device_t next_device;
|
||||
|
||||
pci_set_powerstate(nhi->pci_device, PCI_PM_POWERSTATE_D3);
|
||||
|
||||
// Disable bridges up to and including the upstream-most one managed by the driver.
|
||||
// nhi_init will have stored the first bridge it found (if any) while looking for
|
||||
// the NHI
|
||||
next_device = nhi->pci_device;
|
||||
do {
|
||||
device = next_device;
|
||||
pci_bus_master_enable(device, false);
|
||||
pci_memory_space_enable(device, false);
|
||||
next_device = pci_get_bridge(device);
|
||||
pci_free(device);
|
||||
} while (nhi->upstream_bridge != NULL && device != nhi->upstream_bridge);
|
||||
nhi->upstream_bridge = NULL;
|
||||
nhi->pci_device = NULL;
|
||||
}
|
||||
|
||||
void nhi_quiesce_and_free(nhi_t *nhi)
|
||||
{
|
||||
dart_unmap_page_range(nhi->dart_id, NHI_BOOKKEEPING_IOVM_BASE, NHI_BOOKKEEPING_PAGES);
|
||||
free((void *)nhi->bookkeeping_base);
|
||||
nhi->bookkeeping_base = 0;
|
||||
|
||||
nhi_quiesce_pci(nhi);
|
||||
|
||||
for (unsigned i = 0; i < NHI_MAX_RINGS; i++) {
|
||||
if (nhi->tx_rings[i].buffers != NULL)
|
||||
free(nhi->tx_rings[i].buffers);
|
||||
}
|
||||
|
||||
for (unsigned i = 0; i < NHI_MAX_RINGS; i++) {
|
||||
if (nhi->rx_rings[i].buffers != NULL)
|
||||
free(nhi->rx_rings[i].buffers);
|
||||
}
|
||||
|
||||
free(nhi);
|
||||
|
||||
dprintf(NHID_INIT, "nhi: quiesced\n");
|
||||
}
|
||||
|
||||
static uint64_t nhi_dart_map(nhi_t *nhi, void *addr, size_t bytes)
|
||||
{
|
||||
ASSERT(nhi != NULL);
|
||||
ASSERT(addr != NULL);
|
||||
ASSERT(bytes % DART_PAGE_SIZE == 0);
|
||||
ASSERT(nhi->buffer_space_mapped + bytes > nhi->buffer_space_mapped);
|
||||
|
||||
uint64_t iovmaddr = NHI_BUFFER_IOVM_BASE + nhi->buffer_space_mapped;
|
||||
uint32_t pages = bytes / DART_PAGE_SIZE;
|
||||
|
||||
dart_map_page_range(nhi->dart_id, (uintptr_t)addr, iovmaddr, pages, false);
|
||||
|
||||
nhi->buffer_space_mapped += bytes;
|
||||
|
||||
return iovmaddr;
|
||||
}
|
||||
|
||||
static void *nhi_allocate_bookkeeping(nhi_t *nhi, size_t bytes) {
|
||||
void *allocated;
|
||||
ASSERT(nhi != NULL);
|
||||
ASSERT(nhi->bookkeeping_base != 0);
|
||||
ASSERT(bytes > 0);
|
||||
ASSERT(bytes <= nhi->bookkeeping_size - nhi->bookkeeping_used);
|
||||
|
||||
allocated = (void *)(nhi->bookkeeping_base + nhi->bookkeeping_used);
|
||||
nhi->bookkeeping_used += bytes;
|
||||
|
||||
return allocated;
|
||||
}
|
||||
|
||||
static uint64_t nhi_translate_bookkeeping(nhi_t *nhi, void *addr) {
|
||||
ASSERT(nhi != NULL);
|
||||
|
||||
ASSERT((uintptr_t)addr >= nhi->bookkeeping_base);
|
||||
ASSERT((uintptr_t)addr < nhi->bookkeeping_base + nhi->bookkeeping_used);
|
||||
|
||||
return NHI_BOOKKEEPING_IOVM_BASE + (uintptr_t)addr - nhi->bookkeeping_base;
|
||||
}
|
||||
|
||||
static void nhi_alloc_ring(nhi_t *nhi, nhi_ring_t *ring, uint32_t num_buffers, uint32_t buffer_size)
|
||||
{
|
||||
size_t total_size;
|
||||
|
||||
ASSERT(ring != NULL);
|
||||
ASSERT(ring->buffers == NULL);
|
||||
ASSERT(ring->descriptors == NULL);
|
||||
ASSERT(num_buffers < 0x10000);
|
||||
ASSERT(buffer_size <= 4096);
|
||||
|
||||
ring->num_buffers = num_buffers;
|
||||
ring->buffer_size = buffer_size;
|
||||
|
||||
// rx and tx buffer descriptor have the same size
|
||||
ring->descriptors = nhi_allocate_bookkeeping(nhi, sizeof(nhi_tx_buffer_desc_t) * num_buffers);
|
||||
ring->descriptors_iovm_addr = nhi_translate_bookkeeping(nhi, ring->descriptors);
|
||||
|
||||
total_size = num_buffers * buffer_size;
|
||||
total_size = (total_size + DART_PAGE_SIZE - 1) & ~(DART_PAGE_SIZE - 1);
|
||||
ring->buffers = memalign(total_size, DART_PAGE_SIZE);
|
||||
ring->buffers_iovm_addr = nhi_dart_map(nhi, ring->buffers, total_size);
|
||||
|
||||
}
|
||||
|
||||
void nhi_init_tx_ring(nhi_t *nhi, uint32_t ring_idx, uint32_t num_buffers, uint32_t buffer_size)
|
||||
{
|
||||
nhi_ring_t *ring;
|
||||
uint32_t addr_lo;
|
||||
uint32_t addr_hi;
|
||||
uint32_t flags;
|
||||
|
||||
ASSERT(nhi != NULL);
|
||||
ASSERT(ring_idx < NHI_MAX_RINGS);
|
||||
ASSERT(num_buffers < 0x10000);
|
||||
ASSERT(buffer_size <= 4096);
|
||||
ASSERT(buffer_size % 4 == 0);
|
||||
|
||||
dprintf(NHID_INIT, "nhi: tx ring %u init with %u buffers of %u bytes\n", ring_idx, num_buffers, buffer_size);
|
||||
|
||||
ring = &nhi->tx_rings[ring_idx];
|
||||
|
||||
nhi_alloc_ring(nhi, ring, num_buffers, buffer_size);
|
||||
|
||||
dprintf(NHID_INIT, "nhi: tx ring %u buffers at %p\n", ring_idx, ring->buffers);
|
||||
dprintf(NHID_INIT, "nhi: tx ring %u buffer descriptors at %p\n", ring_idx, ring->descriptors);
|
||||
|
||||
for (uint32_t i = 0; i < num_buffers; i++) {
|
||||
nhi_tx_buffer_desc_t *desc;
|
||||
uint64_t buffer_addr;
|
||||
|
||||
desc = nhi_get_tx_buffer_desc(nhi, ring_idx, i);
|
||||
memset(desc, 0, sizeof(*desc));
|
||||
buffer_addr = nhi_ring_get_buffer_iovm_addr(ring, i);
|
||||
desc->addr_lo = (uint32_t)buffer_addr;
|
||||
desc->addr_hi = (uint32_t)(buffer_addr >> 32);
|
||||
desc->request_status = 1;
|
||||
desc->descriptor_done = 1;
|
||||
}
|
||||
|
||||
addr_lo = (uint32_t)ring->descriptors_iovm_addr;
|
||||
addr_hi = (uint32_t)(ring->descriptors_iovm_addr >> 32);
|
||||
// buffer size of 4096 is expressed as 0
|
||||
nhi_write_reg(nhi, NHI_REG_TX_RING_ADDR_LO(ring_idx), addr_lo);
|
||||
nhi_write_reg(nhi, NHI_REG_TX_RING_ADDR_HI(ring_idx), addr_hi);
|
||||
nhi_write_reg(nhi, NHI_REG_TX_RING_SIZE(ring_idx), num_buffers);
|
||||
|
||||
// for now, only support raw mode rings
|
||||
flags = NHI_TX_TABLE_VALID | NHI_TX_TABLE_RAW_MODE;
|
||||
nhi_write_reg(nhi, NHI_REG_TX_TABLE_FLAGS(ring_idx), flags);
|
||||
}
|
||||
|
||||
void nhi_disable_tx_ring(nhi_t *nhi, uint32_t ring_idx)
|
||||
{
|
||||
ASSERT(nhi != NULL);
|
||||
ASSERT(ring_idx < NHI_MAX_RINGS);
|
||||
|
||||
nhi_write_reg(nhi, NHI_REG_TX_TABLE_FLAGS(ring_idx), 0);
|
||||
}
|
||||
|
||||
void nhi_init_rx_ring(nhi_t *nhi, uint32_t ring_idx, uint32_t num_buffers, uint32_t buffer_size)
|
||||
{
|
||||
nhi_ring_t *ring;
|
||||
uint32_t addr_lo;
|
||||
uint32_t addr_hi;
|
||||
uint32_t sizes;
|
||||
uint32_t flags;
|
||||
|
||||
ASSERT(nhi != NULL);
|
||||
ASSERT(ring_idx < NHI_MAX_RINGS);
|
||||
ASSERT(num_buffers < 0x10000);
|
||||
ASSERT(buffer_size >= 256);
|
||||
ASSERT(buffer_size <= 0x1000);
|
||||
ASSERT(buffer_size % 4 == 0);
|
||||
|
||||
dprintf(NHID_INIT, "nhi: rx ring %u init with %u buffers of %u bytes\n", ring_idx, num_buffers, buffer_size);
|
||||
|
||||
ring = &nhi->rx_rings[ring_idx];
|
||||
|
||||
nhi_alloc_ring(nhi, ring, num_buffers, buffer_size);
|
||||
|
||||
dprintf(NHID_INIT, "nhi: rx ring %u buffers at %p\n", ring_idx, ring->buffers);
|
||||
dprintf(NHID_INIT, "nhi: rx ring %u buffer descriptors at %p\n", ring_idx, ring->descriptors);
|
||||
|
||||
for (uint32_t i = 0; i < num_buffers; i++) {
|
||||
nhi_rx_buffer_desc_t *desc;
|
||||
uint64_t buffer_addr;
|
||||
|
||||
desc = nhi_get_rx_buffer_desc(nhi, ring_idx, i);
|
||||
memset(desc, 0, sizeof(*desc));
|
||||
buffer_addr = nhi_ring_get_buffer_iovm_addr(ring, i);
|
||||
desc->addr_lo = (uint32_t)buffer_addr;
|
||||
desc->addr_hi = (uint32_t)(buffer_addr >> 32);
|
||||
desc->request_status = 1;
|
||||
}
|
||||
|
||||
addr_lo = (uint32_t)ring->descriptors_iovm_addr;
|
||||
addr_hi = (uint32_t)(ring->descriptors_iovm_addr >> 32);
|
||||
// buffer size of 4096 is expressed as 0
|
||||
sizes = NHI_RX_RING_DESC_RING_SIZE_BUFFER_SIZE(num_buffers, buffer_size);
|
||||
nhi_write_reg(nhi, NHI_REG_RX_RING_ADDR_LO(ring_idx), addr_lo);
|
||||
nhi_write_reg(nhi, NHI_REG_RX_RING_ADDR_HI(ring_idx), addr_hi);
|
||||
nhi_write_reg(nhi, NHI_REG_RX_RING_SIZE_BUFFER_SIZE(ring_idx), sizes);
|
||||
|
||||
// Feed all but the last frame to the NHI... keep the last one back because
|
||||
// producer == consumer means an empty ring
|
||||
nhi_write_reg(nhi, NHI_REG_RX_RING_INDEXES(ring_idx), num_buffers - 1);
|
||||
ring->next_buffer_index = 0;
|
||||
|
||||
// for now, accept every PDF value as start and end of frame
|
||||
nhi_write_reg(nhi, NHI_REG_RX_TABLE_PDF_BITMASKS(ring_idx), 0xffffffff);
|
||||
// for now, only support raw mode rings
|
||||
flags = NHI_RX_TABLE_VALID | NHI_RX_TABLE_RAW_MODE;
|
||||
nhi_write_reg(nhi, NHI_REG_RX_TABLE_FLAGS(ring_idx), flags);
|
||||
}
|
||||
|
||||
void nhi_disable_rx_ring(nhi_t *nhi, uint32_t ring_idx)
|
||||
{
|
||||
ASSERT(nhi != NULL);
|
||||
ASSERT(ring_idx < NHI_MAX_RINGS);
|
||||
|
||||
nhi_write_reg(nhi, NHI_REG_RX_TABLE_FLAGS(ring_idx), 0);
|
||||
}
|
||||
|
||||
int32_t nhi_send_sgl(nhi_t *nhi, uint32_t ring_idx, const nhi_sgl_t *sgl, uint8_t sof_pdf, uint8_t eof_pdf)
|
||||
{
|
||||
nhi_ring_t *ring;
|
||||
volatile nhi_tx_buffer_desc_t *desc;
|
||||
uint32_t bytes = 0;
|
||||
uint8_t *buffer;
|
||||
utime_t start_time;
|
||||
|
||||
ASSERT(nhi != NULL);
|
||||
ASSERT(ring_idx < NHI_MAX_RINGS);
|
||||
|
||||
ring = &nhi->tx_rings[ring_idx];
|
||||
desc = nhi_get_tx_buffer_desc(nhi, ring_idx, ring->next_buffer_index);
|
||||
buffer = nhi_ring_get_buffer(ring, ring->next_buffer_index);
|
||||
|
||||
start_time = system_time();
|
||||
while (!desc->descriptor_done) {
|
||||
if (time_has_elapsed(start_time, 2000000))
|
||||
return NHI_STATUS_TIMEOUT;
|
||||
task_yield();
|
||||
platform_memory_barrier();
|
||||
}
|
||||
|
||||
dprintf(NHID_DMA, "nhi: tx ring %u queueing buffer %u (%p) for transmit, pdfs %04x/%04x\n",
|
||||
ring_idx, ring->next_buffer_index, buffer, sof_pdf, eof_pdf);
|
||||
|
||||
while (sgl != NULL) {
|
||||
ASSERT(bytes + sgl->bytes > bytes);
|
||||
ASSERT(bytes + sgl->bytes <= ring->buffer_size);
|
||||
|
||||
memcpy(buffer + bytes, sgl->buffer, sgl->bytes);
|
||||
bytes += sgl->bytes;
|
||||
|
||||
sgl = sgl->next;
|
||||
}
|
||||
|
||||
#if NHID_TXPACKET_DUMP <= DEBUG_LEVEL
|
||||
printf("nhi: tx packet:\n");
|
||||
debug_dump_packet(buffer, bytes);
|
||||
#endif
|
||||
|
||||
// Update the DMA descriptor the NHI will DMA from host memory
|
||||
desc->length = bytes;
|
||||
desc->sof_pdf = sof_pdf;
|
||||
desc->eof_pdf = eof_pdf;
|
||||
desc->descriptor_done = 0;
|
||||
desc->interrupt_enable = 0;
|
||||
desc->request_status = 1;
|
||||
|
||||
// Update our internal accounting of the next free descriptor in the ring
|
||||
ring->next_buffer_index++;
|
||||
if (ring->next_buffer_index == ring->num_buffers)
|
||||
ring->next_buffer_index = 0;
|
||||
|
||||
platform_memory_barrier();
|
||||
|
||||
// And let the NHI know that there's a new descriptor for it to process
|
||||
nhi_write_reg(nhi, NHI_REG_TX_RING_INDEXES(ring_idx), ring->next_buffer_index << 16);
|
||||
|
||||
return NHI_STATUS_OK;
|
||||
}
|
||||
|
||||
int32_t nhi_send_buffer(nhi_t *nhi, uint32_t ring_idx, void *buffer, uint32_t bytes, uint8_t sof_pdf, uint8_t eof_pdf)
|
||||
{
|
||||
nhi_sgl_t sgl;
|
||||
|
||||
sgl.buffer = buffer;
|
||||
sgl.bytes = bytes;
|
||||
sgl.next = NULL;
|
||||
|
||||
return nhi_send_sgl(nhi, ring_idx, &sgl, sof_pdf, eof_pdf);
|
||||
}
|
||||
|
||||
bool nhi_rx_buffer_available(nhi_t *nhi, uint32_t ring_idx)
|
||||
{
|
||||
nhi_ring_t *ring;
|
||||
volatile nhi_rx_buffer_desc_t *next;
|
||||
ASSERT(nhi != NULL);
|
||||
ASSERT(ring_idx < NHI_MAX_RINGS);
|
||||
|
||||
ring = &nhi->rx_rings[ring_idx];
|
||||
|
||||
next = nhi_get_rx_buffer_desc(nhi, ring_idx, ring->next_buffer_index);
|
||||
|
||||
platform_memory_barrier();
|
||||
|
||||
return next->descriptor_done != 0;
|
||||
}
|
||||
|
||||
int32_t nhi_rx_buffer(nhi_t *nhi, uint32_t ring_idx, void *buffer, uint32_t bytes, uint8_t *sof_pdf, uint8_t *eof_pdf)
|
||||
{
|
||||
nhi_ring_t *ring;
|
||||
nhi_rx_buffer_desc_t *next;
|
||||
nhi_rx_buffer_desc_t descriptor;
|
||||
int32_t status = NHI_STATUS_UNKNOWN_ERROR;
|
||||
utime_t start_time;
|
||||
|
||||
ASSERT(nhi != NULL);
|
||||
ASSERT(ring_idx < NHI_MAX_RINGS);
|
||||
|
||||
ring = &nhi->rx_rings[ring_idx];
|
||||
ASSERT(bytes <= ring->buffer_size);
|
||||
|
||||
start_time = system_time();
|
||||
while (!nhi_rx_buffer_available(nhi, ring_idx)) {
|
||||
if (time_has_elapsed(start_time, 2000000))
|
||||
return NHI_STATUS_TIMEOUT;
|
||||
task_yield();
|
||||
}
|
||||
|
||||
// XXX: Are we guaranteed that buffer update is atomic and/or length write never
|
||||
// passes status update write?
|
||||
next = nhi_get_rx_buffer_desc(nhi, ring_idx, ring->next_buffer_index);
|
||||
memcpy(&descriptor, next, sizeof(descriptor));
|
||||
|
||||
platform_memory_barrier();
|
||||
|
||||
if (descriptor.crc_error) {
|
||||
dprintf(DEBUG_CRITICAL, "nhi: rx ring %u CRC error\n", ring_idx);
|
||||
status = NHI_STATUS_CRC_ERROR;
|
||||
goto done;
|
||||
}
|
||||
|
||||
if (descriptor.length > bytes) {
|
||||
dprintf(DEBUG_CRITICAL, "nhi: rx ring %u length overrun (%u vs %u)\n",
|
||||
ring_idx, descriptor.length, bytes);
|
||||
status = NHI_STATUS_INVALID_DESCRIPTOR;
|
||||
goto done;
|
||||
}
|
||||
|
||||
if (sof_pdf != NULL)
|
||||
*sof_pdf = descriptor.sof_pdf;
|
||||
if (eof_pdf != NULL)
|
||||
*eof_pdf = descriptor.eof_pdf;
|
||||
|
||||
status = descriptor.length;
|
||||
memcpy(buffer, nhi_ring_get_buffer(ring, ring->next_buffer_index), descriptor.length);
|
||||
|
||||
dprintf(NHID_DMA, "nhi: ring %u rx packet pdfs %x/%x bytes %u\n", ring_idx,
|
||||
descriptor.sof_pdf, descriptor.eof_pdf, descriptor.length);
|
||||
#if NHID_RXPACKET_DUMP <= DEBUG_LEVEL
|
||||
printf("nhi: rx packet:\n");
|
||||
debug_dump_packet(buffer, descriptor.length);
|
||||
#endif
|
||||
|
||||
done:
|
||||
// Use the buffer we just processed as the free buffer, and give the previous
|
||||
// free buffer to the DMA engine
|
||||
nhi_write_reg(nhi, NHI_REG_RX_RING_INDEXES(ring_idx), ring->next_buffer_index);
|
||||
ring->next_buffer_index++;
|
||||
if (ring->next_buffer_index == ring->num_buffers)
|
||||
ring->next_buffer_index = 0;
|
||||
// Reset the buffer we just processed
|
||||
next->details = 0;
|
||||
next->length = ring->buffer_size;
|
||||
next->request_status = 1;
|
||||
|
||||
return status;
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
#ifndef NHI_PROTOCOL_H
|
||||
#define NHI_PROTOCOL_H
|
||||
|
||||
#define NHI_CLASS_CODE (0x088000)
|
||||
#define NHI_VENDOR_ID (0x8086)
|
||||
|
||||
typedef struct nhi_rx_buffer_desc {
|
||||
uint32_t addr_lo;
|
||||
uint32_t addr_hi;
|
||||
union {
|
||||
struct {
|
||||
uint32_t length:12;
|
||||
uint32_t eof_pdf:4;
|
||||
uint32_t sof_pdf:4;
|
||||
uint32_t crc_error:1;
|
||||
uint32_t descriptor_done:1;
|
||||
uint32_t request_status:1;
|
||||
uint32_t interrupt_enable:1;
|
||||
uint32_t offset:8;
|
||||
};
|
||||
uint32_t details;
|
||||
};
|
||||
uint32_t timestamp;
|
||||
} nhi_rx_buffer_desc_t;
|
||||
|
||||
typedef struct nhi_tx_buffer_desc {
|
||||
uint32_t addr_lo;
|
||||
uint32_t addr_hi;
|
||||
uint32_t length:12;
|
||||
uint32_t eof_pdf:4;
|
||||
uint32_t sof_pdf:4;
|
||||
uint32_t isochronous_dma_enable:1;
|
||||
uint32_t descriptor_done:1;
|
||||
uint32_t request_status:1;
|
||||
uint32_t interrupt_enable:1;
|
||||
uint32_t offset:8;
|
||||
uint32_t fetch_time;
|
||||
} nhi_tx_buffer_desc_t;
|
||||
|
||||
typedef struct nhi_ring_desc {
|
||||
uint32_t addr_lo;
|
||||
uint32_t addr_hi;
|
||||
// The NHI documentation says that controller registers
|
||||
// must be read/written as 32-bit values
|
||||
uint32_t consumer_idx:16;
|
||||
uint32_t producer_idx:16;
|
||||
uint32_t ring_size:16;
|
||||
uint32_t reserved:16;
|
||||
} nhi_ring_desc_t;
|
||||
|
||||
#define NHI_REG_TX_RING_ADDR_LO(ring) (0x00000 + (ring) * 0x10)
|
||||
#define NHI_REG_TX_RING_ADDR_HI(ring) (0x00004 + (ring) * 0x10)
|
||||
#define NHI_REG_TX_RING_INDEXES(ring) (0x00008 + (ring) * 0x10)
|
||||
#define NHI_REG_TX_RING_SIZE(ring) (0x0000C + (ring) * 0x10)
|
||||
#define NHI_REG_RX_RING_ADDR_LO(ring) (0x08000 + (ring) * 0x10)
|
||||
#define NHI_REG_RX_RING_ADDR_HI(ring) (0x08004 + (ring) * 0x10)
|
||||
#define NHI_REG_RX_RING_INDEXES(ring) (0x08008 + (ring) * 0x10)
|
||||
#define NHI_REG_RX_RING_SIZE_BUFFER_SIZE(ring) (0x0800C + (ring) * 0x10)
|
||||
#define NHI_REG_TX_TABLE_FLAGS(ring) (0x19800 + (ring) * 0x20)
|
||||
#define NHI_REG_RX_TABLE_FLAGS(ring) (0x29800 + (ring) * 0x20)
|
||||
#define NHI_REG_RX_TABLE_PDF_BITMASKS(ring) (0x29804 + (ring) * 0x20)
|
||||
|
||||
#define NHI_RX_RING_DESC_RING_SIZE_BUFFER_SIZE(ring_size, buffer_size) \
|
||||
((ring_size & (0xffff)) | (((buffer_size) & (0xfff)) << 16))
|
||||
|
||||
#define NHI_TX_TABLE_VALID (1 << 31)
|
||||
#define NHI_TX_TABLE_RAW_MODE (1 << 30)
|
||||
|
||||
#define NHI_RX_TABLE_VALID (1 << 31)
|
||||
#define NHI_RX_TABLE_RAW_MODE (1 << 30)
|
||||
|
||||
#endif // NHI_PROTOCOL_H
|
||||
@@ -0,0 +1,17 @@
|
||||
# Copyright (C) 2013 Apple Inc. All rights reserved.
|
||||
#
|
||||
# This document is the property of Apple Inc.
|
||||
# It is considered confidential and proprietary.
|
||||
#
|
||||
# This document may not be reproduced or transmitted in any form,
|
||||
# in whole or in part, without the express written permission of
|
||||
# Apple Inc.
|
||||
#
|
||||
|
||||
LOCAL_DIR := $(GET_LOCAL_DIR)
|
||||
|
||||
OPTIONS += WITH_THUNDERBOLT_NHI=1
|
||||
|
||||
ALL_OBJS += \
|
||||
$(LOCAL_DIR)/nhi.o
|
||||
|
||||
Reference in new issue
Block a user