> Date: Tue, 10 Oct 2023 19:40:31 +0200
> From: Stefan Sperling <s...@stsp.name>
> 
> This patch adds enough code to get Elkart Lake devices with PCI
> Vendor ID 8086 and Product ID 4ba0 to attach and pass traffic.
> 
> dwqe0 at pci0 dev 29 function 1 "Intel Elkhart Lake Ethernet" rev 0x11: rev 
> 0x52, address xx:xx:xx:xx:xx:xx
> eephy0 at dwqe0 phy 1: 88E1512 10/100/1000 PHY, rev. 1
> 
> # ifconfig dwqe0 media
> dwqe0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
>         lladdr xx:xx:xx:xx:xx:xx
>         index 5 priority 0 llprio 3
>         media: Ethernet autoselect (1000baseT full-duplex,master)
>         status: active
>         supported media:
>                 media none
>                 media 10baseT
>                 media 10baseT mediaopt full-duplex
>                 media 100baseTX
>                 media 100baseTX mediaopt full-duplex
>                 media 1000baseT
>                 media 1000baseT mediaopt full-duplex
>                 media autoselect
> 
> There is one issue though: Tx speed is currently capped to 64 Mbps in my
> testing, while Rx works at 300 Mbps. I will keep investigating this problem.
> But since 64 is better than 0 and the device seems to be working reliably
> in my testing I think it's worth getting this in and fixing remaining the
> issues in the tree.
> 
> Various other PCI IDs could attach to this but would require more code.
> I don't have access to such hardware. So they remain commented for now.
> There is a second Elkhart Lake Ethernet device in the same box, with
> PCI id 4bb0. However, this interface has no physical Ethernet port
> exposed, and according to Linux there is no PHY attached to the MAC.
> Linux detects this device but fails to atach a PHY.
> 
> ok?

So the GMAC_VERSION #define is simply wrong.  We should commit the
diff attached and drop the sc_core stuff you added below.

A few more nits below...

> diff refs/heads/master refs/heads/dwqe
> commit - ef065493fb463fe122ab7b535932b871830b6de3
> commit + a82b4a31a1787b55291d125b3adfb40fe58779e4
> blob - c6094ca5a57d964784f56124b8d923393b1abb66
> blob + cf64e8ff2018fb0ee1e7e59b29b2bc2dd281aa99
> --- sys/arch/amd64/conf/GENERIC
> +++ sys/arch/amd64/conf/GENERIC
> @@ -556,6 +556,7 @@ lii*      at pci?                         # Attansic L2 
> Ethernet
>  jme* at pci?                         # JMicron JMC250/JMC260 Ethernet
>  bnxt*        at pci?                         # Broadcom BCM573xx, BCM574xx
>  ixl* at pci?                         # Intel Ethernet 700 Series
> +dwqe*        at pci?                         # Intel Elkhart Lake Ethernet
>  mcx* at pci?                         # Mellanox ConnectX-4
>  iavf*        at pci?                         # Intel Ethernet Adaptive VF
>  aq*  at pci?                         # Aquantia aQtion Ethernet
> blob - 7e6afaba34dd35e3bd180055a68741ee3b773486
> blob + 6a4cc88f048d63430ea9eb6080cd98741f7e1612
> --- sys/dev/fdt/if_dwqe_fdt.c
> +++ sys/dev/fdt/if_dwqe_fdt.c
> @@ -111,6 +111,8 @@ dwqe_fdt_attach(struct device *parent, struct device *
>       }
>       sc->sc_dmat = faa->fa_dmat;
>  
> +     sc->sc_core = DWQE_CORE_GMAC;
> +
>       /* Decide GMAC id through address */
>       switch (faa->fa_reg[0].addr) {
>       case 0xfe2a0000:        /* RK3568 */
> blob - 7d260ef46054d6566ef2f81f6cf96dc8de5a5893
> blob + 7ae36ee6e804b7456c296a248d2b058ffe65f722
> --- sys/dev/ic/dwqe.c
> +++ sys/dev/ic/dwqe.c
> @@ -100,7 +100,14 @@ dwqe_attach(struct dwqe_softc *sc)
>       uint32_t version, mode;
>       int i;
>  
> -     version = dwqe_read(sc, GMAC_VERSION);
> +     switch (sc->sc_core) {
> +     case DWQE_CORE_GMAC:
> +             version = dwqe_read(sc, GMAC_VERSION);
> +             break;
> +     case DWQE_CORE_GMAC4:
> +             version = dwqe_read(sc, GMAC4_VERSION);
> +             break;
> +     }
>       printf(": rev 0x%02x, address %s\n", version & GMAC_VERSION_SNPS_MASK,
>           ether_sprintf(sc->sc_lladdr));
>  
> @@ -705,7 +712,7 @@ dwqe_up(struct dwqe_softc *sc)
>  {
>       struct ifnet *ifp = &sc->sc_ac.ac_if;
>       struct dwqe_buf *txb, *rxb;
> -     uint32_t mode, reg, tqs, rqs;
> +     uint32_t mode, reg, fifosz, tqs, rqs;
>       int i;
>  
>       /* Allocate Tx descriptor ring. */
> @@ -793,9 +800,21 @@ dwqe_up(struct dwqe_softc *sc)
>               mode |= GMAC_MTL_CHAN_RX_OP_MODE_RSF;
>       }
>       mode &= ~GMAC_MTL_CHAN_RX_OP_MODE_RQS_MASK;
> -     rqs = (128 << GMAC_MAC_HW_FEATURE1_RXFIFOSIZE(sc->sc_hw_feature[1]) /
> -         256) - 1;
> -     mode |= rqs << GMAC_MTL_CHAN_RX_OP_MODE_RQS_SHIFT;
> +     if (sc->sc_rxfifo_size)
> +             fifosz = sc->sc_rxfifo_size;
> +     else
> +             fifosz = (128 <<
> +                 GMAC_MAC_HW_FEATURE1_RXFIFOSIZE(sc->sc_hw_feature[1]));
> +     rqs = fifosz / 256 - 1;
> +     mode |= (rqs << GMAC_MTL_CHAN_RX_OP_MODE_RQS_SHIFT) &
> +        GMAC_MTL_CHAN_RX_OP_MODE_RQS_MASK;
> +     if (fifosz >= 4096) {
> +             mode |= GMAC_MTL_CHAN_RX_OP_MODE_EHFC; 
> +             mode &= ~GMAC_MTL_CHAN_RX_OP_MODE_RFD_MASK;
> +             mode |= 0x3 << GMAC_MTL_CHAN_RX_OP_MODE_RFD_SHIFT;
> +             mode &= ~GMAC_MTL_CHAN_RX_OP_MODE_RFA_MASK;
> +             mode |= 0x1 << GMAC_MTL_CHAN_RX_OP_MODE_RFA_SHIFT;
> +     }
>       dwqe_write(sc, GMAC_MTL_CHAN_RX_OP_MODE(0), mode);
>  
>       mode = dwqe_read(sc, GMAC_MTL_CHAN_TX_OP_MODE(0));
> @@ -809,9 +828,14 @@ dwqe_up(struct dwqe_softc *sc)
>       mode &= ~GMAC_MTL_CHAN_TX_OP_MODE_TXQEN_MASK;
>       mode |= GMAC_MTL_CHAN_TX_OP_MODE_TXQEN;
>       mode &= ~GMAC_MTL_CHAN_TX_OP_MODE_TQS_MASK;
> -     tqs = (128 << GMAC_MAC_HW_FEATURE1_TXFIFOSIZE(sc->sc_hw_feature[1]) /
> -         256) - 1;
> -     mode |= tqs << GMAC_MTL_CHAN_TX_OP_MODE_TQS_SHIFT;
> +     if (sc->sc_txfifo_size)
> +             fifosz = sc->sc_txfifo_size;
> +     else
> +             fifosz = (128 <<
> +                 GMAC_MAC_HW_FEATURE1_TXFIFOSIZE(sc->sc_hw_feature[1]));
> +     tqs = (fifosz / 256) - 1;
> +     mode |= (tqs << GMAC_MTL_CHAN_TX_OP_MODE_TQS_SHIFT) &
> +         GMAC_MTL_CHAN_TX_OP_MODE_TQS_MASK;
>       dwqe_write(sc, GMAC_MTL_CHAN_TX_OP_MODE(0), mode);
>  
>       reg = dwqe_read(sc, GMAC_QX_TX_FLOW_CTRL(0));
> blob - d35a811571a16d98ca672d17ec188111d4d92132
> blob + d43c9d12b27b2f871fb7105f99a1fe99a32ec9d9
> --- sys/dev/ic/dwqereg.h
> +++ sys/dev/ic/dwqereg.h
> @@ -41,6 +41,7 @@
>  #define GMAC_MAC_HASH_TAB_REG1       0x0014
>  #define GMAC_VERSION         0x0020
>  #define  GMAC_VERSION_SNPS_MASK              0xff
> +#define GMAC4_VERSION                0x0110
>  #define GMAC_INT_MASK                0x003c
>  #define  GMAC_INT_MASK_LPIIM         (1 << 10)
>  #define  GMAC_INT_MASK_PIM           (1 << 3)
> blob - 68d698a50beef00f002d986d7d24f8984c3efe0c
> blob + 8d2c534793f9bfa5f2895ef5a44ec8e927d4fc00
> --- sys/dev/ic/dwqevar.h
> +++ sys/dev/ic/dwqevar.h
> @@ -24,6 +24,11 @@ enum dwqe_phy_mode {
>       DWQE_PHY_MODE_RGMII_RXID,
>  };
>  
> +enum dwqe_core {
> +     DWQE_CORE_GMAC,
> +     DWQE_CORE_GMAC4
> +};
> +
>  struct dwqe_buf {
>       bus_dmamap_t    tb_map;
>       struct mbuf     *tb_m;
> @@ -60,6 +65,7 @@ struct dwqe_softc {
>       int                     sc_link;
>       int                     sc_phyloc;
>       enum dwqe_phy_mode      sc_phy_mode;
> +     enum dwqe_core          sc_core;
>       struct timeout          sc_phy_tick;
>       int                     sc_fixed_link;
>  
> @@ -97,6 +103,8 @@ struct dwqe_softc {
>       int                     sc_pbl;
>       int                     sc_txpbl;
>       int                     sc_rxpbl;
> +     int                     sc_txfifo_size;
> +     int                     sc_rxfifo_size;
>       int                     sc_axi_config;
>       int                     sc_lpi_en;
>       int                     sc_xit_frm;
> blob - 101ed502e76987c27878712b4921cc49f7eb7f59
> blob + 6868eb3591995804dbc332308c9d9629f73c269f
> --- sys/dev/pci/files.pci
> +++ sys/dev/pci/files.pci
> @@ -363,6 +363,10 @@ device   ixl: ether, ifnet, ifmedia, intrmap, stoeplitz
>  attach       ixl at pci
>  file dev/pci/if_ixl.c                ixl
>  
> +# Intel Elkhart Lake Ethernet
> +attach       dwqe at pci with dwqe_pci
> +file dev/pci/if_dwqe_pci.c           dwqe_pci
> +
>  # Neterion Xframe 10 Gigabit ethernet
>  device       xge: ether, ifnet, ifmedia
>  attach       xge  at pci
> blob - /dev/null
> blob + 101b5670451611858f70b95a1082442b47056600 (mode 644)
> --- /dev/null
> +++ sys/dev/pci/if_dwqe_pci.c
> @@ -0,0 +1,155 @@
> +/* $OpenBSD$ */
> +
> +/*
> + * Copyright (c) 2023 Stefan Sperling <s...@openbsd.org>
> + *
> + * Permission to use, copy, modify, and distribute this software for any
> + * purpose with or without fee is hereby granted, provided that the above
> + * copyright notice and this permission notice appear in all copies.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> + */
> +
> +/*
> + * Driver for the Intel Elkhart Lake ethernet controller.
> + */
> +
> +#include <sys/param.h>
> +#include <sys/conf.h>
> +#include <sys/systm.h>
> +#include <sys/device.h>
> +#include <sys/kernel.h>
> +#include <sys/malloc.h>
> +#include <sys/mbuf.h>
> +#include <sys/queue.h>
> +#include <sys/socket.h>
> +#include <sys/sockio.h>
> +#include <sys/timeout.h>
> +#include <sys/task.h>
> +
> +#include <dev/pci/pcireg.h>
> +#include <dev/pci/pcivar.h>
> +#include <dev/pci/pcidevs.h>
> +
> +#if NBPFILTER > 0
> +#include <net/bpf.h>
> +#endif
> +#include <net/if.h>
> +#include <net/if_dl.h>
> +#include <net/if_media.h>
> +
> +#include <netinet/in.h>
> +#include <netinet/if_ether.h>
> +
> +#include <dev/mii/mii.h>
> +#include <dev/mii/miivar.h>
> +
> +#include <dev/ic/dwqereg.h>
> +#include <dev/ic/dwqevar.h>
> +
> +static const struct pci_matchid dwqe_pci_devices[] = {
> +     { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EHL_PSE0_RGMII_1G },
> +#if 0
> +     { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EHL_PSE0_SGMII_1G },
> +     { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EHL_PSE0_SGMII_2G },
> +     { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EHL_PSE1_RGMII_1G },
> +     { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EHL_PSE1_SGMII_1G },
> +     { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EHL_PSE1_SGMII_2G },
> +#endif
> +};
> +
> +struct dwqe_pci_softc {
> +     struct dwqe_softc       sc_sc;
> +     pci_chipset_tag_t       sc_pct;
> +     pcitag_t                sc_pcitag;
> +     bus_size_t              sc_mapsize;
> +};
> +
> +int
> +dwqe_pci_match(struct device *parent, void *cfdata, void *aux)
> +{
> +     struct pci_attach_args *pa = aux;
> +     return pci_matchbyid(pa, dwqe_pci_devices, nitems(dwqe_pci_devices));
> +}
> +
> +void
> +dwqe_pci_attach(struct device *parent, struct device *self, void *aux)
> +{
> +     struct pci_attach_args *pa = aux;
> +     struct dwqe_pci_softc *psc = (void *)self;
> +     struct dwqe_softc *sc = &psc->sc_sc;
> +     pci_intr_handle_t ih;
> +     pcireg_t memtype;
> +     int err;
> +     const char *intrstr;
> +
> +     psc->sc_pct = pa->pa_pc;
> +     psc->sc_pcitag = pa->pa_tag;
> +
> +     sc->sc_dmat = pa->pa_dmat;
> +
> +     memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_MAPREG_START);
> +     err = pci_mapreg_map(pa, PCI_MAPREG_START, memtype, 0,
> +         &sc->sc_iot, &sc->sc_ioh, NULL, &psc->sc_mapsize, 0);
> +     if (err) {
> +             printf("%s: can't map mem space\n", DEVNAME(sc));
> +             return;
> +     }
> +
> +     if (pci_intr_map_msi(pa, &ih) && pci_intr_map(pa, &ih)) {
> +             printf("%s: can't map interrupt\n", DEVNAME(sc));
> +             return;
> +     }
> +
> +     intrstr = pci_intr_string(psc->sc_pct, ih);
> +     sc->sc_ih = pci_intr_establish(pa->pa_pc, ih, IPL_NET | IPL_MPSAFE,
> +         dwqe_intr, psc, sc->sc_dev.dv_xname);
> +     if (sc->sc_ih == NULL) {
> +             printf(": can't establish interrupt");
> +             if (intrstr != NULL)
> +                     printf(" at %s", intrstr);
> +             printf("\n");
> +             return;
> +     }
> +
> +     switch (PCI_PRODUCT(pa->pa_id)) {
> +     case PCI_PRODUCT_INTEL_EHL_PSE0_RGMII_1G:
> +             sc->sc_phy_mode = DWQE_PHY_MODE_RGMII;

This suggests that the PHY must be programmed to disable its internal
delays.  Our eephy(4) driver doesn't do this.  That may explain the
poor performance.  See how MIIF_SETDELAY, MIIF_RXID and MIIF_TXID are
used in rgephy(4).

Anyway, that is a different driver, so a separate diff that can go in
after this one.

> +             sc->sc_clk = GMAC_MAC_MDIO_ADDR_CR_250_300; 
> +             sc->sc_clkrate = 200000000;
> +             break;
> +     default:
> +             sc->sc_phy_mode = DWQE_PHY_MODE_UNKNOWN;
> +             break;
> +     }
> +
> +     sc->sc_phyloc = MII_PHY_ANY;
> +     sc->sc_core = DWQE_CORE_GMAC4;
> +     sc->sc_8xpbl = 1;
> +     sc->sc_txpbl = 32;
> +     sc->sc_rxpbl = 32;
> +     sc->sc_txfifo_size = 4096 * 8;
> +     sc->sc_rxfifo_size = 4096 * 8;

Makes more sense to simply use 32768 here.  Intel's data sheet says 32KB.

> +
> +     sc->sc_axi_config = 1;
> +     sc->sc_wr_osr_lmt = 1;
> +     sc->sc_rd_osr_lmt = 1;
> +     sc->sc_blen[0] = 4;
> +     sc->sc_blen[1] = 8;
> +     sc->sc_blen[2] = 16;
> +
> +     dwqe_lladdr_read(sc, sc->sc_lladdr);
> +
> +     dwqe_reset(sc);
> +     dwqe_attach(sc);
> +}
> +
> +const struct cfattach dwqe_pci_ca = {
> +     sizeof(struct dwqe_softc), dwqe_pci_match, dwqe_pci_attach
> +};
> 
> 


Index: dev/ic/dwqereg.h
===================================================================
RCS file: /cvs/src/sys/dev/ic/dwqereg.h,v
retrieving revision 1.2
diff -u -p -r1.2 dwqereg.h
--- dev/ic/dwqereg.h    16 Feb 2023 14:43:53 -0000      1.2
+++ dev/ic/dwqereg.h    10 Oct 2023 18:40:19 -0000
@@ -39,8 +39,6 @@
 #define  GMAC_MAC_PACKET_FILTER_PR     (1 << 0)
 #define GMAC_MAC_HASH_TAB_REG0 0x0010
 #define GMAC_MAC_HASH_TAB_REG1 0x0014
-#define GMAC_VERSION           0x0020
-#define  GMAC_VERSION_SNPS_MASK                0xff
 #define GMAC_INT_MASK          0x003c
 #define  GMAC_INT_MASK_LPIIM           (1 << 10)
 #define  GMAC_INT_MASK_PIM             (1 << 3)
@@ -60,6 +58,8 @@
 #define GMAC_INT_STATUS                0x00b0
 #define GMAC_INT_EN            0x00b4
 #define GMAC_MAC_1US_TIC_CTR   0x00dc
+#define GMAC_VERSION           0x0110
+#define  GMAC_VERSION_SNPS_MASK                0xff
 #define GMAC_MAC_HW_FEATURE(x) (0x011c + (x) * 0x4)
 #define  GMAC_MAC_HW_FEATURE1_TXFIFOSIZE(x) (((x) >> 6) & 0x1f)
 #define  GMAC_MAC_HW_FEATURE1_RXFIFOSIZE(x) (((x) >> 0) & 0x3f)

Reply via email to