/* * Copyright (c) 2015, 2016, 2017, 2022, 2023 Jonas 'Sortie' Termansen. * Copyright (c) 2015 Meisaka Yukara. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * * net/em/em.cpp * 825xx driver. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../arp.h" #include "../ether.h" #include "em.h" #include "emregs.h" namespace Sortix { namespace EM { static const int RECEIVE_PACKET_COUNT = 32; static const int FEATURE_EEPROM = 1 << 0; // EEPROM access present static const int FEATURE_SERDES = 1 << 1; // SerDes/TBI supported static const int FEATURE_PCIE = 1 << 2; // PCIe Device enum feature_index { emdefault, em8254xS, em8254xM, em8256xM, em8257xM, em8257xS, em82576S, em8258xM, em8258xS, emfeaturemax }; static const int feature_table[emfeaturemax] = { [emdefault] = 0, [em8254xS] = FEATURE_EEPROM | FEATURE_SERDES, [em8254xM] = FEATURE_EEPROM, [em8256xM] = FEATURE_PCIE, [em8257xM] = FEATURE_PCIE, [em8257xS] = FEATURE_SERDES | FEATURE_PCIE, [em82576S] = FEATURE_SERDES | FEATURE_PCIE, [em8258xM] = FEATURE_PCIE, [em8258xS] = FEATURE_SERDES | FEATURE_PCIE, }; struct device { uint16_t feature_index; uint16_t device_id; }; static const struct device device_table[] = { { em8254xS, PCI_PRODUCT_INTEL_DH89XXCC_SGMII }, { em8254xS, PCI_PRODUCT_INTEL_DH89XXCC_S }, { em8254xS, PCI_PRODUCT_INTEL_DH89XXCC_BPLANE }, { em8254xS, PCI_PRODUCT_INTEL_DH89XXCC_SFP }, { em8254xS, PCI_PRODUCT_INTEL_82542 }, { em8254xS, PCI_PRODUCT_INTEL_82543GC_F }, { em8254xS, PCI_PRODUCT_INTEL_82543GC_C }, { em8254xS, PCI_PRODUCT_INTEL_82544EI_C }, { em8254xS, PCI_PRODUCT_INTEL_82544EI_F }, { em8254xS, PCI_PRODUCT_INTEL_82544GC_C }, { em8254xM, PCI_PRODUCT_INTEL_82544GC_LOM }, { em8254xM, PCI_PRODUCT_INTEL_82540EM_D }, { em8254xS, PCI_PRODUCT_INTEL_82545EM_C }, { em8254xS, PCI_PRODUCT_INTEL_82546EB_C }, { em8254xS, PCI_PRODUCT_INTEL_82545EM_F }, { em8254xS, PCI_PRODUCT_INTEL_82546EB_F }, { em8254xM, PCI_PRODUCT_INTEL_82541EI_C }, { em8254xM, PCI_PRODUCT_INTEL_82541ER_LOM }, { em8254xM, PCI_PRODUCT_INTEL_82540EM_M }, { em8254xM, PCI_PRODUCT_INTEL_82540EP_M }, { em8254xM, PCI_PRODUCT_INTEL_82540EP_D }, { em8254xM, PCI_PRODUCT_INTEL_82541EI_M }, { em8254xM, PCI_PRODUCT_INTEL_82547EI }, { em8254xM, PCI_PRODUCT_INTEL_82547EI_M }, { em8254xS, PCI_PRODUCT_INTEL_82546EB_CQ }, { em8254xS, PCI_PRODUCT_INTEL_82540EP_LP }, { em8254xS, PCI_PRODUCT_INTEL_82545GM_C }, { em8254xS, PCI_PRODUCT_INTEL_82545GM_F }, { em8254xS, PCI_PRODUCT_INTEL_82545GM_S }, { em8256xM, PCI_PRODUCT_INTEL_ICH8_IGP_M_AMT }, { em8256xM, PCI_PRODUCT_INTEL_ICH8_IGP_AMT }, { em8256xM, PCI_PRODUCT_INTEL_ICH8_IGP_C }, { em8256xM, PCI_PRODUCT_INTEL_ICH8_IFE }, { em8256xM, PCI_PRODUCT_INTEL_ICH8_IGP_M }, { em8257xS, PCI_PRODUCT_INTEL_82571EB_C }, { em8257xS, PCI_PRODUCT_INTEL_82571EB_F }, { em8257xS, PCI_PRODUCT_INTEL_82571EB_S }, { em8254xS, PCI_PRODUCT_INTEL_82547GI }, { em8254xM, PCI_PRODUCT_INTEL_82541GI_C }, { em8254xM, PCI_PRODUCT_INTEL_82541GI_M }, { em8254xM, PCI_PRODUCT_INTEL_82541ER_C }, { em8254xS, PCI_PRODUCT_INTEL_82546GB_C }, { em8254xS, PCI_PRODUCT_INTEL_82546GB_F }, { em8254xS, PCI_PRODUCT_INTEL_82546GB_S }, { em8254xM, PCI_PRODUCT_INTEL_82541GI_LF }, { em8257xS, PCI_PRODUCT_INTEL_82572EI_C }, { em8257xS, PCI_PRODUCT_INTEL_82572EI_F }, { em8257xS, PCI_PRODUCT_INTEL_82572EI_S }, { em8254xS, PCI_PRODUCT_INTEL_82546GB_PCIE }, { em8257xM, PCI_PRODUCT_INTEL_82573E }, { em8257xM, PCI_PRODUCT_INTEL_82573E_IAMT }, { em8257xM, PCI_PRODUCT_INTEL_82573E_IDE }, { em8257xM, PCI_PRODUCT_INTEL_82573E_KCS }, { em8257xM, PCI_PRODUCT_INTEL_82573E_SERIAL }, { em8257xS, PCI_PRODUCT_INTEL_80003ES2LAN_CD }, { em8257xS, PCI_PRODUCT_INTEL_80003ES2LAN_SD }, { em8254xS, PCI_PRODUCT_INTEL_82546GB_CQ }, { em8257xM, PCI_PRODUCT_INTEL_82573L }, { em8254xS, PCI_PRODUCT_INTEL_82546GB_2 }, { em8257xS, PCI_PRODUCT_INTEL_82571EB_AT }, { em8257xS, PCI_PRODUCT_INTEL_82571EB_AF }, { em8257xS, PCI_PRODUCT_INTEL_82571EB_CQ }, { em8257xS, PCI_PRODUCT_INTEL_82571EB_FQ }, { em8257xS, PCI_PRODUCT_INTEL_82575EB_C }, { em8257xS, PCI_PRODUCT_INTEL_82575EB_S }, { em8257xM, PCI_PRODUCT_INTEL_82573L_PL_1 }, { em8257xM, PCI_PRODUCT_INTEL_82573V_PM }, { em8257xM, PCI_PRODUCT_INTEL_82573E_PM }, { em8257xM, PCI_PRODUCT_INTEL_82573L_PL_2 }, { em8254xS, PCI_PRODUCT_INTEL_82546GB_CQ_K }, { em8257xS, PCI_PRODUCT_INTEL_82572EI }, { em8257xS, PCI_PRODUCT_INTEL_80003ES2LAN_C }, { em8257xS, PCI_PRODUCT_INTEL_80003ES2LAN_S }, { em8257xS, PCI_PRODUCT_INTEL_82571EB_CQ_LP }, { em8256xM, PCI_PRODUCT_INTEL_ICH9_IGP_AMT }, { em8256xM, PCI_PRODUCT_INTEL_ICH9_IGP_M }, { em8256xM, PCI_PRODUCT_INTEL_ICH9_IFE }, { em8256xM, PCI_PRODUCT_INTEL_ICH9_IFE_G }, { em8256xM, PCI_PRODUCT_INTEL_ICH9_IFE_GT }, { em8256xM, PCI_PRODUCT_INTEL_ICH8_IFE_GT }, { em8256xM, PCI_PRODUCT_INTEL_ICH8_IFE_G }, { em82576S, PCI_PRODUCT_INTEL_82576 }, { em8256xM, PCI_PRODUCT_INTEL_ICH9_IGP_M_V }, { em8256xM, PCI_PRODUCT_INTEL_ICH10_R_BM_LM }, { em8256xM, PCI_PRODUCT_INTEL_ICH10_R_BM_LF }, { em8256xM, PCI_PRODUCT_INTEL_ICH10_R_BM_V }, { em8257xM, PCI_PRODUCT_INTEL_82574L }, { em8257xS, PCI_PRODUCT_INTEL_82571PT_CQ }, { em8257xS, PCI_PRODUCT_INTEL_82575GB_CQ }, { em8257xS, PCI_PRODUCT_INTEL_82571EB_SD }, { em8257xS, PCI_PRODUCT_INTEL_82571EB_SQ }, { em8256xM, PCI_PRODUCT_INTEL_ICH10_D_BM_LM }, { em8256xM, PCI_PRODUCT_INTEL_ICH10_D_BM_LF }, { em8257xS, PCI_PRODUCT_INTEL_82575GB_QP_PM }, { em8256xM, PCI_PRODUCT_INTEL_ICH9_BM }, { em82576S, PCI_PRODUCT_INTEL_82576_F }, { em82576S, PCI_PRODUCT_INTEL_82576_S }, { em82576S, PCI_PRODUCT_INTEL_82576_CQ }, { em8257xM, PCI_PRODUCT_INTEL_82577LM }, { em8257xM, PCI_PRODUCT_INTEL_82577LC }, { em8257xS, PCI_PRODUCT_INTEL_82578DM }, { em8257xS, PCI_PRODUCT_INTEL_82578DC }, { em8256xM, PCI_PRODUCT_INTEL_ICH9_IGP_M_AMT }, { em8257xM, PCI_PRODUCT_INTEL_82574LA }, { em8254xS, PCI_PRODUCT_INTEL_82544EI_A4 }, { em8254xS, PCI_PRODUCT_INTEL_82544GC_A4 }, { em8256xM, PCI_PRODUCT_INTEL_ICH8_82567V_3 }, { em8257xM, PCI_PRODUCT_INTEL_82579LM }, { em8257xM, PCI_PRODUCT_INTEL_82579V }, { em82576S, PCI_PRODUCT_INTEL_82576_NS }, { em8258xS, PCI_PRODUCT_INTEL_82583V }, { em82576S, PCI_PRODUCT_INTEL_82576_SQ }, { em8258xS, PCI_PRODUCT_INTEL_82580_C }, { em8258xS, PCI_PRODUCT_INTEL_82580_F }, { em8258xS, PCI_PRODUCT_INTEL_82580_S }, { em8258xS, PCI_PRODUCT_INTEL_82580_SGMII }, { em8258xS, PCI_PRODUCT_INTEL_82580_CD }, { em82576S, PCI_PRODUCT_INTEL_82576_NS_S }, { em82576S, PCI_PRODUCT_INTEL_I350_C }, { em82576S, PCI_PRODUCT_INTEL_I350_F }, { em82576S, PCI_PRODUCT_INTEL_I350_S }, { em82576S, PCI_PRODUCT_INTEL_I350_SGMII }, { em82576S, PCI_PRODUCT_INTEL_82576_CQ_ET2 }, { em8258xS, PCI_PRODUCT_INTEL_82580_FQ }, { em8257xS, PCI_PRODUCT_INTEL_I210_C }, { em8257xS, PCI_PRODUCT_INTEL_I210_F }, { em8257xS, PCI_PRODUCT_INTEL_I210_S }, { em8257xS, PCI_PRODUCT_INTEL_I210_SGMII }, { em8257xS, PCI_PRODUCT_INTEL_I211_C }, { em8257xM, PCI_PRODUCT_INTEL_I217_LM }, { em8257xM, PCI_PRODUCT_INTEL_I217_V }, { em8257xM, PCI_PRODUCT_INTEL_I218_V }, { em8257xM, PCI_PRODUCT_INTEL_I218_LM }, { em8257xS, PCI_PRODUCT_INTEL_I210_C_NF }, { em8257xS, PCI_PRODUCT_INTEL_I210_S_NF }, { em8257xM, PCI_PRODUCT_INTEL_I218_LM_2 }, { em8257xM, PCI_PRODUCT_INTEL_I218_V_2 }, { em8257xM, PCI_PRODUCT_INTEL_I218_LM_3 }, { em8257xM, PCI_PRODUCT_INTEL_I218_V_3 }, { em8256xM, PCI_PRODUCT_INTEL_ICH9_IGP_C }, { em8254xM, PCI_PRODUCT_INTEL_EP80579_LAN_1 }, { em8254xM, PCI_PRODUCT_INTEL_EP80579_LAN_4 }, { em8254xM, PCI_PRODUCT_INTEL_EP80579_LAN_2 }, { em8254xM, PCI_PRODUCT_INTEL_EP80579_LAN_5 }, { em8254xM, PCI_PRODUCT_INTEL_EP80579_LAN_3 }, { em8254xM, PCI_PRODUCT_INTEL_EP80579_LAN_6 } }; static const size_t device_table_count = sizeof(device_table) / sizeof(device_table[0]); static const uint32_t understood_interrupts = EM_INTERRUPT_TXDW | EM_INTERRUPT_TXQE | EM_INTERRUPT_LSC | EM_INTERRUPT_RXDMT0 | EM_INTERRUPT_RXO | EM_INTERRUPT_RXT0 | EM_INTERRUPT_MDAC | EM_INTERRUPT_RXCFG | EM_INTERRUPT_TXD_LOW | EM_INTERRUPT_SRPD; struct rx_desc { little_uint64_t address; little_uint16_t length; little_uint16_t checksum; little_uint8_t status; little_uint8_t errors; little_uint16_t special; }; struct tx_desc_tcpdata { little_uint64_t address; little_uint32_t lencmd; little_uint8_t status; little_uint8_t opts; little_uint16_t special; }; class EM : public NetworkInterface { public: EM(uint32_t devaddr, size_t number, int features); virtual ~EM(); public: virtual bool Send(Ref pkt); bool Initialize(); private: bool Reset(); __attribute__((format(printf, 2, 3))) void Log(const char* format, ...); uint32_t Read32(uint32_t reg); void Write32(uint32_t reg, uint32_t value); bool ReadEEPROM(uint16_t* out, uint16_t reg); bool ReadPHY(uint16_t* out, uint8_t reg); bool WritePHY(uint8_t reg, uint16_t value); bool WaitLinkResolved(); void RegisterInterrupts(); bool AddReceiveDescriptor(Ref pkt); bool AddTransmitDescriptor(Ref pkt); bool CanAddTransmit(); static void InterruptHandler(struct interrupt_context*, void*); static void InterruptWorkHandler(void* context); void OnInterrupt(); void InterruptWork(); private: uint32_t devaddr; struct interrupt_handler interrupt_registration; struct interrupt_work interrupt_work; uint32_t interrupt_work_icr; uint8_t interrupt; addralloc_t mmio_alloc; volatile uint8_t* mmio_base; paddrmapped_t rdesc_alloc; paddrmapped_t tdesc_alloc; int features; struct rx_desc* rdesc; struct tx_desc_tcpdata* tdesc; Ref* rpackets; Ref* tpackets; Ref tx_queue_first; Ref tx_queue_last; kthread_mutex_t tx_lock; kthread_mutex_t eeprom_lock; kthread_mutex_t phy_lock; uint32_t rx_count; uint32_t tx_count; uint32_t rx_tail; uint32_t rx_prochead; uint32_t tx_tail; uint32_t tx_prochead; }; void EM::Log(const char* format, ...) { Log::PrintF("%s: ", ifinfo.name); va_list ap; va_start(ap, format); Log::PrintFV(format, ap); va_end(ap); Log::PrintF("\n"); } EM::EM(uint32_t devaddr, size_t number, int features) { snprintf(ifinfo.name, sizeof(ifinfo.name), "em%zu", number); ifinfo.type = IF_TYPE_ETHERNET; ifinfo.features = IF_FEATURE_ETHERNET_CRC_OFFLOAD; ifinfo.addrlen = ETHER_ADDR_LEN; ifstatus.mtu = ETHERMTU; this->devaddr = devaddr; interrupt = 0; memset(&interrupt_registration, 0, sizeof(interrupt_registration)); interrupt_work.handler = InterruptWorkHandler; interrupt_work.context = this; interrupt_work_icr = 0; memset(&mmio_alloc, 0, sizeof(mmio_alloc)); mmio_base = NULL; memset(&rdesc_alloc, 0, sizeof(rdesc_alloc)); memset(&tdesc_alloc, 0, sizeof(tdesc_alloc)); this->features = features; rdesc = NULL; tdesc = NULL; rpackets = NULL; tpackets = NULL; // tx_queue_first has constructor. // tx_queue_last has constructor. tx_lock = KTHREAD_MUTEX_INITIALIZER; eeprom_lock = KTHREAD_MUTEX_INITIALIZER; phy_lock = KTHREAD_MUTEX_INITIALIZER; rx_count = 0; tx_count = 0; rx_tail = 0; rx_prochead = 0; tx_tail = 0; tx_prochead = 0; } EM::~EM() { assert(false); } uint32_t EM::Read32(uint32_t reg) { return *((volatile little_uint32_t*) (mmio_base + reg)); } void EM::Write32(uint32_t reg, uint32_t value) { *((volatile little_uint32_t*) (mmio_base + reg)) = value; } bool EM::ReadEEPROM(uint16_t* out, uint16_t reg) { ScopedLock lock(&eeprom_lock); uint32_t value = ((uint32_t) reg) << EM_MAIN_REG_EERD_ADDR_SHIFT | EM_MAIN_REG_EERD_START; Write32(EM_MAIN_REG_EERD, value); struct timespec now = Time::Get(CLOCK_MONOTONIC); struct timespec end = timespec_add(now, timespec_make(1, 0)); while ( true ) { now = Time::Get(CLOCK_MONOTONIC); if ( (value = Read32(EM_MAIN_REG_EERD)) & EM_MAIN_REG_EERD_DONE ) break; if ( timespec_le(end, now) ) return false; } *out = (value & EM_MAIN_REG_EERD_DATA_MASK) >> EM_MAIN_REG_EERD_DATA_SHIFT; return true; } bool EM::ReadPHY(uint16_t* out, uint8_t reg) { ScopedLock lock(&phy_lock); uint32_t mdic = Read32(EM_MAIN_REG_MDIC); if ( mdic & EM_MAIN_REG_MDIC_E ) return false; mdic = ((uint32_t) reg) << EM_MAIN_REG_MDIC_REGADD_SHIFT | EM_MAIN_REG_MDIC_PHYADD_PHY_ONE_AND_ONLY | EM_MAIN_REG_MDIC_OP_READ; Write32(EM_MAIN_REG_MDIC, mdic); uint32_t stop_bits = EM_MAIN_REG_MDIC_R | EM_MAIN_REG_MDIC_E; struct timespec now = Time::Get(CLOCK_MONOTONIC); struct timespec end = timespec_add(now, timespec_make(1, 0)); while ( true ) { now = Time::Get(CLOCK_MONOTONIC); if ( (mdic = Read32(EM_MAIN_REG_MDIC)) & stop_bits ) break; if ( timespec_le(end, now) ) return false; } if ( mdic & EM_MAIN_REG_MDIC_E ) return false; *out = (mdic & EM_MAIN_REG_MDIC_DATA_MASK) >> EM_MAIN_REG_MDIC_DATA_SHIFT; return true; } bool EM::WritePHY(uint8_t reg, uint16_t value) { ScopedLock lock(&phy_lock); uint32_t mdic = Read32(EM_MAIN_REG_MDIC); if ( mdic & EM_MAIN_REG_MDIC_E ) return false; mdic = ((uint32_t) value) << EM_MAIN_REG_MDIC_DATA_SHIFT | ((uint32_t) reg) << EM_MAIN_REG_MDIC_REGADD_SHIFT | EM_MAIN_REG_MDIC_PHYADD_PHY_ONE_AND_ONLY | EM_MAIN_REG_MDIC_OP_WRITE; Write32(EM_MAIN_REG_MDIC, mdic); uint32_t stop_bits = EM_MAIN_REG_MDIC_R | EM_MAIN_REG_MDIC_E; struct timespec now = Time::Get(CLOCK_MONOTONIC); struct timespec end = timespec_add(now, timespec_make(1, 0)); while ( true ) { now = Time::Get(CLOCK_MONOTONIC); if ( (mdic = Read32(EM_MAIN_REG_MDIC)) & stop_bits ) break; if ( timespec_le(end, now) ) return false; } if ( mdic & EM_MAIN_REG_MDIC_E ) return false; return true; } bool EM::WaitLinkResolved() { uint16_t pstatus; uint16_t psstat; struct timespec now = Time::Get(CLOCK_MONOTONIC); struct timespec end = timespec_add(now, timespec_make(1, 0)); do { now = Time::Get(CLOCK_MONOTONIC); if ( !ReadPHY(&pstatus, EM_PHY_REG_PSTATUS) || !ReadPHY(&psstat, EM_PHY_REG_PSSTAT) ) return Log("error: WaitLinkResolved failed to read PHY"), false; if ( !(psstat & EM_PHY_REG_PSSTAT_LINK) ) return false; if ( !(pstatus & EM_PHY_REG_PSTATUS_AN_COMPLETE) ) continue; // TODO: Needed? if ( !(psstat & EM_PHY_REG_PSSTAT_SPEED_DUPLEX_RESOLVED) ) continue; return true; } while ( timespec_lt(now, end) ); Log("error: WaitLinkResolved timed out"); return false; } bool EM::AddReceiveDescriptor(Ref pkt) // ordered via interrupt worker { uint32_t next_desc = rx_tail + 1; if ( rx_count <= next_desc ) next_desc = 0; if ( next_desc == rx_prochead ) return false; struct rx_desc* desc = &rdesc[rx_tail]; memset(desc, 0, sizeof(*desc)); desc->status = 0; desc->address = pkt->pmap.phys; rpackets[rx_tail] = pkt; rx_tail = next_desc; // TODO: Research whether this is needed, or whether the paging bits do // the right thing. Do those bits work on all systems? //asm volatile ("wbinvd"); Write32(EM_MAIN_REG_RDT, rx_tail); return true; } bool EM::AddTransmitDescriptor(Ref pkt) // tx_lock must be locked. { uint32_t next_desc = tx_tail + 1; if ( tx_count <= next_desc ) next_desc = 0; if ( next_desc == tx_prochead ) return false; struct tx_desc_tcpdata* desc = &tdesc[tx_tail]; desc->address = pkt->pmap.phys; desc->lencmd = EM_TDESC_LENGTH(pkt->length) | EM_TDESC_TYPE_TCPDATA | EM_TDESC_CMD_RS | EM_TDESC_CMD_EOP | EM_TDESC_CMD_IFCS; desc->status = 0; desc->opts = 0; desc->special = 0; tpackets[tx_tail] = pkt; tx_tail = next_desc; // TODO: Research whether this is needed, or whether the paging bits do // the right thing. Do those bits work on all systems? //asm volatile ("wbinvd"); Write32(EM_MAIN_REG_TDT, tx_tail); return true; } bool EM::CanAddTransmit() // tx_lock must be locked. { uint32_t next_desc = tx_tail + 1; if ( tx_count <= next_desc ) next_desc = 0; if ( next_desc == tx_prochead ) return false; return true; } bool EM::Send(Ref pkt) { ScopedLock lock(&tx_lock); if ( !tx_queue_first && CanAddTransmit() && AddTransmitDescriptor(pkt) ) return true; if ( tx_queue_last ) { tx_queue_last->next = pkt; tx_queue_last = pkt; } else { tx_queue_first = pkt; tx_queue_last = pkt; } pkt->next.Reset(); return true; } void EM::InterruptHandler(struct interrupt_context*, void* user) { ((EM*) user)->OnInterrupt(); } void EM::RegisterInterrupts() { interrupt = PCI::GetInterruptIndex(devaddr); interrupt_registration.handler = EM::InterruptHandler; interrupt_registration.context = this; Interrupt::RegisterHandler(interrupt, &interrupt_registration); } void EM::OnInterrupt() { // TODO: Use "Interrupt Acknowledge Auto Mask Register - IAM (000E0h)" to // automatically mask set interrupts. uint32_t icr = Read32(EM_MAIN_REG_ICR); if ( !icr ) return; if ( interrupt_work_icr != 0 ) { #if 0 // TODO: This assertion fired! Is it fine to just ignore the interrupt? // Does reading ICR have side effects? Should we read it at the // of the interrupt work handler and reschedule the interrupt // worker? if ( interrupt_work_icr ) PanicF("em: icr=%#x but interrupt_work_icr=%#x\n", icr, interrupt_work_icr); assert(interrupt_work_icr == 0); #endif return; } interrupt_work_icr = icr; // Mask further interrupts while processing the interrupt. Write32(EM_MAIN_REG_IMC, understood_interrupts); Interrupt::ScheduleWork(&interrupt_work); } void EM::InterruptWorkHandler(void* context) { ((EM*) context)->InterruptWork(); } void EM::InterruptWork() { uint32_t icr = interrupt_work_icr; uint32_t unhandled = icr & 0x1FFFF; if ( icr & EM_INTERRUPT_LSC ) { // TODO: This can block the kernel worker thread for a second. WaitLinkResolved(); uint32_t status = Read32(EM_MAIN_REG_STATUS); ScopedLock lock(&cfg_lock); if ( status & EM_MAIN_REG_STATUS_LU ) ifstatus.flags |= IF_STATUS_FLAGS_UP; else ifstatus.flags &= ~IF_STATUS_FLAGS_UP; kthread_cond_broadcast(&cfg_cond); poll_channel.Signal(PollEventStatus()); unhandled &= ~EM_INTERRUPT_LSC; } if ( icr & EM_INTERRUPT_RXDMT0 ) { // Add more receive descriptors when we run out faster than we can // process the incoming packets. for ( size_t i = 0; i < RECEIVE_PACKET_COUNT; i++ ) { Ref buf = GetPacket(); if ( buf ) { if ( !AddReceiveDescriptor(buf) ) break; } } unhandled &= ~EM_INTERRUPT_RXDMT0; } if ( icr & EM_INTERRUPT_MDAC ) { // MDI/O Access Complete } if ( icr & EM_INTERRUPT_RXT0 ) { // Receive timer expired, check descriptors. while ( rx_prochead != rx_tail && rdesc[rx_prochead].status ) { Ref rxpacket = rpackets[rx_prochead]; rpackets[rx_prochead].Reset(); assert(rxpacket.IsUnique()); rxpacket->length = rdesc[rx_prochead].length; assert(rxpacket->pmap.phys == rdesc[rx_prochead].address); rxpacket->netif = this; Ether::Handle(rxpacket, true); rxpacket.Reset(); rx_prochead++; if ( rx_count <= rx_prochead ) rx_prochead = 0; if ( rx_prochead != rx_tail ) { Ref buf = GetPacket(); // TODO: Design a solution that handles when there's no more // packets available, but later adds packets when they // become available, otherwise the receive queue might // deadlock with no available packets. if ( buf ) AddReceiveDescriptor(buf); } } unhandled &= ~EM_INTERRUPT_RXT0; } if ( icr & EM_INTERRUPT_RXO ) { // TODO: Receiver overrun, do we need more buffers? unhandled &= ~EM_INTERRUPT_RXO; } kthread_mutex_lock(&tx_lock); if ( icr & (EM_INTERRUPT_TXDW | EM_INTERRUPT_TXD_LOW) ) { // Transmit descriptor written back. // Transmit descriptor low threshold. while ( tx_prochead != tx_tail && (tdesc[tx_prochead].status & EM_RDESC_STATUS_DD) ) { tpackets[tx_prochead].Reset(); tx_prochead++; if ( tx_count <= tx_prochead ) tx_prochead = 0; } unhandled &= ~(EM_INTERRUPT_TXDW | EM_INTERRUPT_TXD_LOW); } if ( icr & EM_INTERRUPT_TXQE ) { // Transmit queue is empty. Head should equal tail. assert(tx_tail < tx_count); while ( tx_prochead != tx_tail ) { tpackets[tx_prochead].Reset(); tx_prochead++; if ( tx_count <= tx_prochead ) tx_prochead = 0; } unhandled &= ~EM_INTERRUPT_TXQE; } while ( tx_queue_first && CanAddTransmit() ) { Ref pkt = tx_queue_first; tx_queue_first = pkt->next; pkt->next.Reset(); AddTransmitDescriptor(pkt); } kthread_mutex_unlock(&tx_lock); interrupt_work_icr = 0; // Unmask interrupts so they can be delivered again. Write32(EM_MAIN_REG_IMS, understood_interrupts); } bool EM::Reset() { uint32_t ctrl; uint32_t status; PCI::DisableBusMaster(devaddr); PCI::DisableInterruptLine(devaddr); if ( features & FEATURE_PCIE ) { // For PCIe devices, disable GIO Master prior to reset. ctrl = Read32(EM_MAIN_REG_CTRL); ctrl |= EM_MAIN_REG_CTRL_GIOMD; Write32(EM_MAIN_REG_CTRL, ctrl); struct timespec now = Time::Get(CLOCK_MONOTONIC); struct timespec end = timespec_add(now, timespec_make(1, 0)); while ( true ) { now = Time::Get(CLOCK_MONOTONIC); status = Read32(EM_MAIN_REG_STATUS); if ( !(status & EM_MAIN_REG_STATUS_GIOME) ) break; if ( timespec_le(end, now) ) { Log("error: Failed to disable GIO Master prior to reset"); return false; } } } // Clear all interrupts and disable rx/tx. Write32(EM_MAIN_REG_IMC, UINT32_MAX); Write32(EM_MAIN_REG_RCTL, 0); Write32(EM_MAIN_REG_TCTL, 0); // Reset the device, this initializes everything to default settings ctrl = Read32(EM_MAIN_REG_CTRL); ctrl |= EM_MAIN_REG_CTRL_RST; Write32(EM_MAIN_REG_CTRL, ctrl); // TODO: The documentation mentioned waiting a short interval here before // checking the register again. // Wait for it to finish. struct timespec now = Time::Get(CLOCK_MONOTONIC); struct timespec end = timespec_add(now, timespec_make(1, 0)); while ( true ) { // Can exit this wait if card is done loading it's settings. if ( Read32(EM_MAIN_REG_EECD) & EM_MAIN_REG_EECD_ARD ) break; // Hack to make sure the control read below is valid. // On some hardware, this loop would hang without this. // Read all the statisics registers (which we do later anyway). for ( uint32_t x = 0; x < 256; x += 4 ) Read32(EM_STAT_REG_CRCERRS + x); // Read, and wait for reset to finish. ctrl = Read32(EM_MAIN_REG_CTRL); if ( !(ctrl & EM_MAIN_REG_CTRL_PHY_RST) ) break; if ( timespec_le(end, now) ) { Log("error: Timed out waiting for device reset"); return false; } } // Disable interrupts after the reset. Write32(EM_MAIN_REG_IMC, UINT32_MAX); if ( features & FEATURE_EEPROM ) { // If we have EEPROM read the MAC directly from it. uint16_t macw[3]; if ( !ReadEEPROM(&macw[0], EM_EEPROM_REG_ETHERNET_ADDR_1) || !ReadEEPROM(&macw[1], EM_EEPROM_REG_ETHERNET_ADDR_2) || !ReadEEPROM(&macw[2], EM_EEPROM_REG_ETHERNET_ADDR_3) ) return Log("error: Failed to read EEPROM"), false; ifinfo.addr[0] = macw[0] >> 0 & 0xFF; ifinfo.addr[1] = macw[0] >> 8 & 0xFF; ifinfo.addr[2] = macw[1] >> 0 & 0xFF; ifinfo.addr[3] = macw[1] >> 8 & 0xFF; ifinfo.addr[4] = macw[2] >> 0 & 0xFF; ifinfo.addr[5] = macw[2] >> 8 & 0xFF; } else { uint32_t macd[2]; // Receive Address[0] is programmed with the hardware mac from PROM or // EEPROM after the device is reset. macd[0] = Read32(EM_FILTER_REG_RAL); macd[1] = Read32(EM_FILTER_REG_RAH); ifinfo.addr[0] = macd[0] >> 0 & 0xFF; ifinfo.addr[1] = macd[0] >> 8 & 0xFF; ifinfo.addr[2] = macd[0] >> 16 & 0xFF; ifinfo.addr[3] = macd[0] >> 24 & 0xFF; ifinfo.addr[4] = macd[1] >> 0 & 0xFF; ifinfo.addr[5] = macd[1] >> 8 & 0xFF; } memcpy(&cfg.ether.address, ifinfo.addr, sizeof(struct ether_addr)); // Enable bus mastering so the card can read/write memory. PCI::EnableBusMaster(devaddr); PCI::EnableMemoryWrite(devaddr); status = Read32(EM_MAIN_REG_STATUS); bool inserdes = false; if ( features & FEATURE_SERDES ) inserdes = status & EM_MAIN_REG_STATUS_TBIMODE; ctrl = Read32(EM_MAIN_REG_CTRL); if ( inserdes ) ctrl &= ~EM_MAIN_REG_CTRL_LRST; // TBI/SerDes only. ctrl |= EM_MAIN_REG_CTRL_ASDE; ctrl |= EM_MAIN_REG_CTRL_SLU; ctrl &= ~EM_MAIN_REG_CTRL_ILOS; ctrl &= ~EM_MAIN_REG_CTRL_FRCSPD; ctrl &= ~EM_MAIN_REG_CTRL_FRCDPLX; ctrl &= ~EM_MAIN_REG_CTRL_VME; // TODO: CTRL.RFCE (hub 8/9/10 pdf 11.4.3.2 says read from phy regs) // TODO: CTRL.TFCE (hub 8/9/10 pdf 11.4.3.2 says read from phy regs) // TODO: CTRL.ILOS Write32(EM_MAIN_REG_CTRL, ctrl); // CTRL.FRCSPD = CTRL.FRCDPLX = 0b; CTRL.ASDE = 1b // CTRL.FD Duplex if FRCDPLX is set, ignored otherwise // CTRL.SLU Enable link // CTRL.ASDE Auto-Speed Detection Enable, ignored in TBI/Serdes mode // CTRL.RFCE respond to reception of flow control packets. // Set by Auto-negotiation if negotiation is enabled. // CTRL.TFCE Ethernet controller transmits flow control packets // based on the receive FIFO fullness, or when triggered. // Set by Auto-negotiation if negotiation is enabled. // CTRL.ILOS Invert Loss-of-Signal, reserved on some devices. // set to 0. // CTRL.SPEED Speed if FRCSPD is set, ignored otherwise. // CTRL.VME Enable VLAN Tag removal and processing. // STATUS.FD Reflects the value of CTRL.FD as above. // STATUS.LU Reflects internal link status // STATUS.SPEED Speed status bits reflect speed resolved from ASD function. /* For the 82541xx and 82547GI/EI, configure the LED behavior through LEDCTRL. TODO */ // FCAH and FCAL should contain the flow control Ethernet address. // 01:80:C2:00:00:01 and ethertype 0x8808 Write32(EM_MAIN_REG_FCAH, 0x0100); Write32(EM_MAIN_REG_FCAL, 0x00c28001); Write32(EM_MAIN_REG_FCT, 0x8808); Write32(EM_MAIN_REG_FCTTV, 0); // Clear all statistical counters. for ( uint32_t x = 0; x < 256; x += 4 ) Read32(EM_STAT_REG_CRCERRS + x); // Setup the descriptor tables, Write32(EM_MAIN_REG_RCTL, 0); Write32(EM_MAIN_REG_TCTL, 0); rx_tail = 0; rx_prochead = 0; tx_tail = 0; tx_prochead = 0; rx_count = rdesc_alloc.size / sizeof(struct rx_desc); tx_count = tdesc_alloc.size / sizeof(struct tx_desc_tcpdata); rdesc = (struct rx_desc*) rdesc_alloc.from; tdesc = (struct tx_desc_tcpdata*) tdesc_alloc.from; if ( !rpackets ) rpackets = new Ref[rx_count]; if ( !tpackets ) tpackets = new Ref[tx_count]; if ( !rpackets || !tpackets ) return Log("error: Failed to allocate packets: %m"), false; Write32(EM_MAIN_REG_RDLEN, rdesc_alloc.size); Write32(EM_MAIN_REG_RDH, 0); Write32(EM_MAIN_REG_RDT, 0); Write32(EM_MAIN_REG_RDBAL, (uint64_t) rdesc_alloc.phys & 0xffffffff); Write32(EM_MAIN_REG_RDBAH, (uint64_t) rdesc_alloc.phys >> 32); Write32(EM_MAIN_REG_RADV, 0); Write32(EM_MAIN_REG_RSRPD, 0); Write32(EM_MAIN_REG_TXDCTL, EM_MAIN_REG_TXDCTL_WTHRESH(1) | EM_MAIN_REG_TXDCTL_GRAN); uint32_t tipg = Read32(EM_MAIN_REG_TIPG); // TODO: Is programming TIPG needed? tipg = EM_MAIN_REG_TIPG_IPGT(10) | EM_MAIN_REG_TIPG_IPGR1(4) | EM_MAIN_REG_TIPG_IPGR2(6); tipg = EM_MAIN_REG_TIPG_IPGT(8) | /* For 82567 */ EM_MAIN_REG_TIPG_IPGR1(8) | EM_MAIN_REG_TIPG_IPGR2(7); // Was 0x902008 on my laptop Write32(EM_MAIN_REG_TIPG, tipg); Write32(EM_MAIN_REG_TDLEN, tdesc_alloc.size); Write32(EM_MAIN_REG_TDH, 0); Write32(EM_MAIN_REG_TDT, 0); Write32(EM_MAIN_REG_TDBAL, (uint64_t) tdesc_alloc.phys & 0xffffffff); Write32(EM_MAIN_REG_TDBAH, (uint64_t) tdesc_alloc.phys >> 32); for ( size_t i = 0; i < RECEIVE_PACKET_COUNT; i++ ) { Ref buf = GetPacket(); if ( !buf ) { if ( !i ) return Log("error: Failed to allocate packets: %m"), false; break; } if ( !AddReceiveDescriptor(buf) ) break; } // Enable Receive and Transmit. Write32(EM_MAIN_REG_RCTL, EM_MAIN_REG_RCTL_EN | EM_MAIN_REG_RCTL_SBP | EM_MAIN_REG_RCTL_MPE | EM_MAIN_REG_RCTL_BAM | EM_MAIN_REG_RCTL_SECRC); Write32(EM_MAIN_REG_TCTL, EM_MAIN_REG_TCTL_EN | EM_MAIN_REG_TCTL_PSP | EM_MAIN_REG_TCTL_CT(15) | EM_MAIN_REG_TCTL_COLD(64) | EM_MAIN_REG_TCTL_RTLC /**/| EM_MAIN_REG_TCTL_RESERVED1); // Check if the link is already up since it might not send an interrupt. status = Read32(EM_MAIN_REG_STATUS); { ScopedLock lock(&cfg_lock); if ( status & EM_MAIN_REG_STATUS_LU ) ifstatus.flags |= IF_STATUS_FLAGS_UP; else ifstatus.flags &= ~IF_STATUS_FLAGS_UP; kthread_cond_broadcast(&cfg_cond); poll_channel.Signal(PollEventStatus()); } RegisterInterrupts(); PCI::EnableInterruptLine(devaddr); // Reset all the interrupt status (set all interrupts). Write32(EM_MAIN_REG_IMS, UINT32_MAX); // Disable all interrupts. Write32(EM_MAIN_REG_IMC, UINT32_MAX); // Enable relevant interrupts. Write32(EM_MAIN_REG_IMS, understood_interrupts); return true; } bool EM::Initialize() { pcibar_t mmio_bar = PCI::GetBAR(devaddr, 0); if ( mmio_bar.size() < 128*1024 ) { Log("error: Register area is too small"); return errno = EINVAL, false; } if ( !MapPCIBAR(&mmio_alloc, mmio_bar, Memory::PAT_UC) ) { Log("error: Registers could not be mapped: %m"); return false; } mmio_base = (volatile uint8_t*) mmio_alloc.from; if ( !AllocateAndMapPage(&rdesc_alloc, PAGE_USAGE_DRIVER, Memory::PAT_UC) ) { Log("error: Failed to map descriptor page"); return false; } if ( !AllocateAndMapPage(&tdesc_alloc, PAGE_USAGE_DRIVER, Memory::PAT_UC) ) { FreeAllocatedAndMappedPage(&rdesc_alloc); Log("error: Failed to map descriptor page"); return false; } return Reset(); } struct Search { const char* devpath; Ref dev; size_t number; }; static bool InitializeDevice(uint32_t devaddr, const pciid_t* /*id*/, const pcitype_t* /*type*/, void* u, void* info) { Search* search = (Search*) u; int features = *(const int*) info; EM* em = new EM(devaddr, search->number++, features); if ( !em ) return true; if ( !em->Initialize() ) return true; if ( !RegisterNetworkInterface(em, search->dev) ) PanicF("%s: Failed to register as network interface", em->ifinfo.name); return true; } void Init(const char* devpath, Ref dev) { Search search = { .devpath = devpath, .dev = dev, .number = 0 }; pcifind_t* filters = new pcifind_t[device_table_count]; if ( !filters ) Panic("em: Failed to allocate PCI filters"); for ( size_t i = 0; i < device_table_count; i++ ) { const int* features = &feature_table[device_table[i].feature_index]; uint32_t device_id = device_table[i].device_id; filters[i] = pcifind_t((void*) features, PCI_VENDOR_INTEL, device_id, 0x02, 0x00, 0x00); } PCI::Search(InitializeDevice, &search, filters, device_table_count); delete[] filters; } } // namespace EM } // namespace Sortix