From 3d8eeacc236d8a32e3bcc572c1f4bed4cb03a40d Mon Sep 17 00:00:00 2001
From: Munehisa Kamata <kamatam@amazon.com>
Date: Thu, 27 Jul 2017 22:09:37 +0000
Subject: drivers/amazon: ena: update to 1.2.0
Update to 1.2.0 by importing the version from the internal tree.
The source files are based on branch 'ena_version_1.2.0' of the internal
ena driver repository.
Signed-off-by: Munehisa Kamata <kamatam@amazon.com>
Reviewed-by: Anchal Agarwal <anchalag@amazon.com>
Reviewed-by: Guru Anbalagane <guruanb@amazon.com>
Reviewed-by: Eduardo Valentin <eduval@amazon.com>
CR: https://cr.amazon.com/r/7475891/
---
drivers/amazon/Documentation/networking/ena.txt | 5 +-
drivers/amazon/net/ena/ena_com.c | 67 ++--
drivers/amazon/net/ena/ena_com.h | 5 +-
drivers/amazon/net/ena/ena_eth_com.c | 5 +
drivers/amazon/net/ena/ena_ethtool.c | 16 +-
drivers/amazon/net/ena/ena_netdev.c | 402 ++++++++++++++++--------
drivers/amazon/net/ena/ena_netdev.h | 27 +-
drivers/amazon/net/ena/ena_regs_defs.h | 32 ++
8 files changed, 373 insertions(+), 186 deletions(-)
diff --git a/drivers/amazon/Documentation/networking/ena.txt b/drivers/amazon/Documentation/networking/ena.txt
index 2b4b6f5..3a9cff1 100644
--- a/drivers/amazon/Documentation/networking/ena.txt
+++ b/drivers/amazon/Documentation/networking/ena.txt
@@ -129,8 +129,9 @@ The ENA driver supports two Queue Operation modes for Tx SQs:
memory.
- Low Latency Queue (LLQ) mode or "push-mode".
* In this mode the driver pushes the transmit descriptors and the
- first 128 bytes of the packet directly to the ENA device memory
- space. The rest of the packet payload is fetched by the
+ first few bytes of the packet (negotiable parameter)
+ directly to the ENA device memory space.
+ The rest of the packet payload is fetched by the
device. For this operation mode, the driver uses a dedicated PCI
device memory BAR, which is mapped with write-combine capability.
diff --git a/drivers/amazon/net/ena/ena_com.c b/drivers/amazon/net/ena/ena_com.c
index f2e167b..52beba8 100644
--- a/drivers/amazon/net/ena/ena_com.c
+++ b/drivers/amazon/net/ena/ena_com.c
@@ -61,6 +61,8 @@
#define ENA_MMIO_READ_TIMEOUT 0xFFFFFFFF
+#define ENA_REGS_ADMIN_INTR_MASK 1
+
/*****************************************************************************/
/*****************************************************************************/
/*****************************************************************************/
@@ -97,8 +99,8 @@ static inline int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
return -EINVAL;
}
- ena_addr->mem_addr_low = (u32)addr;
- ena_addr->mem_addr_high = (u64)addr >> 32;
+ ena_addr->mem_addr_low = lower_32_bits(addr);
+ ena_addr->mem_addr_high = (u16)upper_32_bits(addr);
return 0;
}
@@ -232,11 +234,9 @@ static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queu
tail_masked = admin_queue->sq.tail & queue_size_mask;
/* In case of queue FULL */
- cnt = admin_queue->sq.tail - admin_queue->sq.head;
+ cnt = atomic_read(&admin_queue->outstanding_cmds);
if (cnt >= admin_queue->q_depth) {
- pr_debug("admin queue is FULL (tail %d head %d depth: %d)\n",
- admin_queue->sq.tail, admin_queue->sq.head,
- admin_queue->q_depth);
+ pr_debug("admin queue is full.\n");
admin_queue->stats.out_of_space++;
return ERR_PTR(-ENOSPC);
}
@@ -494,7 +494,7 @@ static int ena_com_comp_status_to_errno(u8 comp_status)
case ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE:
return -ENOMEM;
case ENA_ADMIN_UNSUPPORTED_OPCODE:
- return -EPERM;
+ return -EOPNOTSUPP;
case ENA_ADMIN_BAD_OPCODE:
case ENA_ADMIN_MALFORMED_REQUEST:
case ENA_ADMIN_ILLEGAL_PARAMETER:
@@ -513,7 +513,14 @@ static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_c
timeout = jiffies + usecs_to_jiffies(admin_queue->completion_timeout);
- while (comp_ctx->status == ENA_CMD_SUBMITTED) {
+ while (1) {
+ spin_lock_irqsave(&admin_queue->q_lock, flags);
+ ena_com_handle_admin_completion(admin_queue);
+ spin_unlock_irqrestore(&admin_queue->q_lock, flags);
+
+ if (comp_ctx->status != ENA_CMD_SUBMITTED)
+ break;
+
if (time_is_before_jiffies(timeout)) {
pr_err("Wait for completion (polling) timeout\n");
/* ENA didn't have any completion */
@@ -526,10 +533,6 @@ static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_c
goto err;
}
- spin_lock_irqsave(&admin_queue->q_lock, flags);
- ena_com_handle_admin_completion(admin_queue);
- spin_unlock_irqrestore(&admin_queue->q_lock, flags);
-
msleep(100);
}
@@ -599,13 +602,14 @@ static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
volatile struct ena_admin_ena_mmio_req_read_less_resp *read_resp =
mmio_read->read_resp;
- u32 mmio_read_reg, timeout, ret;
+ u32 mmio_read_reg, ret, i;
unsigned long flags;
- int i;
+ u32 timeout = mmio_read->reg_read_to;
might_sleep();
- timeout = mmio_read->reg_read_to ? : ENA_REG_READ_TIMEOUT;
+ if (timeout == 0)
+ timeout = ENA_REG_READ_TIMEOUT;
/* If readless is disabled, perform regular read */
if (!mmio_read->readless_supported)
@@ -786,7 +790,7 @@ static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev,
if (!ena_com_check_supported_feature_id(ena_dev, feature_id)) {
pr_debug("Feature %d isn't supported\n", feature_id);
- return -EPERM;
+ return -EOPNOTSUPP;
}
memset(&get_cmd, 0x0, sizeof(get_cmd));
@@ -1324,7 +1328,7 @@ int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
if ((get_resp.u.aenq.supported_groups & groups_flag) != groups_flag) {
pr_warn("Trying to set unsupported aenq events. supported flag: %x asked flag: %x\n",
get_resp.u.aenq.supported_groups, groups_flag);
- return -EPERM;
+ return -EOPNOTSUPP;
}
memset(&cmd, 0x0, sizeof(cmd));
@@ -1456,6 +1460,12 @@ void ena_com_admin_destroy(struct ena_com_dev *ena_dev)
void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling)
{
+ u32 mask_value = 0;
+
+ if (polling)
+ mask_value = ENA_REGS_ADMIN_INTR_MASK;
+
+ writel(mask_value, ena_dev->reg_bar + ENA_REGS_INTR_MASK_OFF);
ena_dev->admin_queue.polling = polling;
}
@@ -1732,7 +1742,7 @@ int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
if (!rc)
memcpy(&get_feat_ctx->hw_hints, &get_resp.u.hw_hints,
sizeof(get_resp.u.hw_hints));
- else if (rc == -EPERM)
+ else if (rc == -EOPNOTSUPP)
memset(&get_feat_ctx->hw_hints, 0x0,
sizeof(get_feat_ctx->hw_hints));
else
@@ -1815,7 +1825,8 @@ void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data)
writel((u32)aenq->head, dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
}
-int ena_com_dev_reset(struct ena_com_dev *ena_dev)
+int ena_com_dev_reset(struct ena_com_dev *ena_dev,
+ enum ena_regs_reset_reason_types reset_reason)
{
u32 stat, timeout, cap, reset_val;
int rc;
@@ -1843,6 +1854,8 @@ int ena_com_dev_reset(struct ena_com_dev *ena_dev)
/* start reset */
reset_val = ENA_REGS_DEV_CTL_DEV_RESET_MASK;
+ reset_val |= (reset_reason << ENA_REGS_DEV_CTL_RESET_REASON_SHIFT) &
+ ENA_REGS_DEV_CTL_RESET_REASON_MASK;
writel(reset_val, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
/* Write again the MMIO read request address */
@@ -1925,7 +1938,7 @@ int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu)
if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_MTU)) {
pr_debug("Feature %d isn't supported\n", ENA_ADMIN_MTU);
- return -EPERM;
+ return -EOPNOTSUPP;
}
memset(&cmd, 0x0, sizeof(cmd));
@@ -1979,7 +1992,7 @@ int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
ENA_ADMIN_RSS_HASH_FUNCTION)) {
pr_debug("Feature %d isn't supported\n",
ENA_ADMIN_RSS_HASH_FUNCTION);
- return -EPERM;
+ return -EOPNOTSUPP;
}
/* Validate hash function is supported */
@@ -1991,7 +2004,7 @@ int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
if (get_resp.u.flow_hash_func.supported_func & (1 << rss->hash_func)) {
pr_err("Func hash %d isn't supported by device, abort\n",
rss->hash_func);
- return -EPERM;
+ return -EOPNOTSUPP;
}
memset(&cmd, 0x0, sizeof(cmd));
@@ -2050,7 +2063,7 @@ int ena_com_fill_hash_function(struct ena_com_dev *ena_dev,
if (!((1 << func) & get_resp.u.flow_hash_func.supported_func)) {
pr_err("Flow hash function %d isn't supported\n", func);
- return -EPERM;
+ return -EOPNOTSUPP;
}
switch (func) {
@@ -2143,7 +2156,7 @@ int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
ENA_ADMIN_RSS_HASH_INPUT)) {
pr_debug("Feature %d isn't supported\n",
ENA_ADMIN_RSS_HASH_INPUT);
- return -EPERM;
+ return -EOPNOTSUPP;
}
memset(&cmd, 0x0, sizeof(cmd));
@@ -2224,7 +2237,7 @@ int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev)
pr_err("hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n",
i, hash_ctrl->supported_fields[i].fields,
hash_ctrl->selected_fields[i].fields);
- return -EPERM;
+ return -EOPNOTSUPP;
}
}
@@ -2302,7 +2315,7 @@ int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
ena_dev, ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG)) {
pr_debug("Feature %d isn't supported\n",
ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
- return -EPERM;
+ return -EOPNOTSUPP;
}
ret = ena_com_ind_tbl_convert_to_device(ena_dev);
@@ -2569,7 +2582,7 @@ int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev)
ENA_ADMIN_INTERRUPT_MODERATION);
if (rc) {
- if (rc == -EPERM) {
+ if (rc == -EOPNOTSUPP) {
pr_debug("Feature %d isn't supported\n",
ENA_ADMIN_INTERRUPT_MODERATION);
rc = 0;
diff --git a/drivers/amazon/net/ena/ena_com.h b/drivers/amazon/net/ena/ena_com.h
index f8cdce0..7b784f8 100644
--- a/drivers/amazon/net/ena/ena_com.h
+++ b/drivers/amazon/net/ena/ena_com.h
@@ -234,6 +234,7 @@ struct ena_com_stats_admin {
struct ena_com_admin_queue {
void *q_dmadev;
spinlock_t q_lock; /* spinlock for the admin queue */
+
struct ena_comp_ctx *comp_ctx;
u32 completion_timeout;
u16 q_depth;
@@ -419,10 +420,12 @@ void ena_com_admin_destroy(struct ena_com_dev *ena_dev);
/* ena_com_dev_reset - Perform device FLR to the device.
* @ena_dev: ENA communication layer struct
+ * @reset_reason: Specify what is the trigger for the reset in case of an error.
*
* @return - 0 on success, negative value on failure.
*/
-int ena_com_dev_reset(struct ena_com_dev *ena_dev);
+int ena_com_dev_reset(struct ena_com_dev *ena_dev,
+ enum ena_regs_reset_reason_types reset_reason);
/* ena_com_create_io_queue - Create io queue.
* @ena_dev: ENA communication layer struct
diff --git a/drivers/amazon/net/ena/ena_eth_com.c b/drivers/amazon/net/ena/ena_eth_com.c
index f999305..b11e573 100644
--- a/drivers/amazon/net/ena/ena_eth_com.c
+++ b/drivers/amazon/net/ena/ena_eth_com.c
@@ -493,6 +493,11 @@ int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id)
if (cdesc_phase != expected_phase)
return -EAGAIN;
+ if (unlikely(cdesc->req_id >= io_cq->q_depth)) {
+ pr_err("Invalid req id %d\n", cdesc->req_id);
+ return -EINVAL;
+ }
+
ena_com_cq_inc_head(io_cq);
*req_id = READ_ONCE(cdesc->req_id);
diff --git a/drivers/amazon/net/ena/ena_ethtool.c b/drivers/amazon/net/ena/ena_ethtool.c
index 58361d6..b1212de 100644
--- a/drivers/amazon/net/ena/ena_ethtool.c
+++ b/drivers/amazon/net/ena/ena_ethtool.c
@@ -93,6 +93,8 @@ static const struct ena_stats ena_stats_rx_strings[] = {
ENA_STAT_RX_ENTRY(dma_mapping_err),
ENA_STAT_RX_ENTRY(bad_desc_num),
ENA_STAT_RX_ENTRY(rx_copybreak_pkt),
+ ENA_STAT_RX_ENTRY(bad_req_id),
+ ENA_STAT_RX_ENTRY(empty_rx_ring),
};
static const struct ena_stats ena_stats_ena_com_strings[] = {
@@ -538,12 +540,8 @@ static int ena_get_rss_hash(struct ena_com_dev *ena_dev,
}
rc = ena_com_get_hash_ctrl(ena_dev, proto, &hash_fields);
- if (rc) {
- /* If device don't have permission, return unsupported */
- if (rc == -EPERM)
- rc = -EOPNOTSUPP;
+ if (rc)
return rc;
- }
cmd->data = ena_flow_hash_to_flow_type(hash_fields);
@@ -611,7 +609,7 @@ static int ena_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info)
rc = -EOPNOTSUPP;
}
- return (rc == -EPERM) ? -EOPNOTSUPP : rc;
+ return rc;
}
static int ena_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info,
@@ -637,7 +635,7 @@ static int ena_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info,
rc = -EOPNOTSUPP;
}
- return (rc == -EPERM) ? -EOPNOTSUPP : rc;
+ return rc;
}
static u32 ena_get_rxfh_indir_size(struct net_device *netdev)
@@ -738,7 +736,6 @@ static int ena_set_rxfh(struct net_device *netdev, const u32 *indir,
return 0;
}
-#ifndef HAVE_RHEL6_ETHTOOL_OPS_EXT_STRUCT
static void ena_get_channels(struct net_device *netdev,
struct ethtool_channels *channels)
@@ -755,7 +752,6 @@ static void ena_get_channels(struct net_device *netdev,
channels->combined_count = 0;
}
-#endif /* HAVE_RHEL6_ETHTOOL_OPS_EXT_STRUCT */
static int ena_get_tunable(struct net_device *netdev,
const struct ethtool_tunable *tuna, void *data)
{
@@ -817,9 +813,7 @@ static const struct ethtool_ops ena_ethtool_ops = {
.get_rxfh_key_size = ena_get_rxfh_key_size,
.get_rxfh = ena_get_rxfh,
.set_rxfh = ena_set_rxfh,
-#ifndef HAVE_RHEL6_ETHTOOL_OPS_EXT_STRUCT
.get_channels = ena_get_channels,
-#endif
.get_tunable = ena_get_tunable,
.set_tunable = ena_set_tunable,
};
diff --git a/drivers/amazon/net/ena/ena_netdev.c b/drivers/amazon/net/ena/ena_netdev.c
index 82add9d..e4b670b 100644
--- a/drivers/amazon/net/ena/ena_netdev.c
+++ b/drivers/amazon/net/ena/ena_netdev.c
@@ -87,6 +87,7 @@ static void ena_tx_timeout(struct net_device *dev)
if (test_and_set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
return;
+ adapter->reset_reason = ENA_REGS_RESET_OS_NETDEV_WD;
u64_stats_update_begin(&adapter->syncp);
adapter->dev_stats.tx_timeout++;
u64_stats_update_end(&adapter->syncp);
@@ -109,11 +110,10 @@ static int ena_change_mtu(struct net_device *dev, int new_mtu)
if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) {
netif_err(adapter, drv, dev,
- "Invalid MTU setting. new_mtu: %d\n", new_mtu);
-
+ "Invalid MTU setting. new_mtu: %d max mtu: %d min mtu: %d\n",
+ new_mtu, adapter->max_mtu, ENA_MIN_MTU);
return -EINVAL;
}
-
ret = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
if (!ret) {
netif_dbg(adapter, drv, dev, "set MTU to %d\n", new_mtu);
@@ -140,7 +140,7 @@ static int ena_init_rx_cpu_rmap(struct ena_adapter *adapter)
int irq_idx = ENA_IO_IRQ_IDX(i);
rc = irq_cpu_rmap_add(adapter->netdev->rx_cpu_rmap,
- adapter->msix_entries[irq_idx].vector);
+ pci_irq_vector(adapter->pdev, irq_idx));
if (rc) {
free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);
adapter->netdev->rx_cpu_rmap = NULL;
@@ -197,6 +197,7 @@ static void ena_init_io_rings(struct ena_adapter *adapter)
rxr->sgl_size = adapter->max_rx_sgl_size;
rxr->smoothed_interval =
ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
+ rxr->empty_rx_queue = 0;
}
}
@@ -309,6 +310,24 @@ static void ena_free_all_io_tx_resources(struct ena_adapter *adapter)
ena_free_tx_resources(adapter, i);
}
+static inline int validate_rx_req_id(struct ena_ring *rx_ring, u16 req_id)
+{
+ if (likely(req_id < rx_ring->ring_size))
+ return 0;
+
+ netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
+ "Invalid rx req_id: %hu\n", req_id);
+
+ u64_stats_update_begin(&rx_ring->syncp);
+ rx_ring->rx_stats.bad_req_id++;
+ u64_stats_update_end(&rx_ring->syncp);
+
+ /* Trigger device reset */
+ rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
+ set_bit(ENA_FLAG_TRIGGER_RESET, &rx_ring->adapter->flags);
+ return -EFAULT;
+}
+
/* ena_setup_rx_resources - allocate I/O Rx resources (Descriptors)
* @adapter: network interface device structure
* @qid: queue index
@@ -320,7 +339,7 @@ static int ena_setup_rx_resources(struct ena_adapter *adapter,
{
struct ena_ring *rx_ring = &adapter->rx_ring[qid];
struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
- int size, node;
+ int size, node, i;
if (rx_ring->rx_buffer_info) {
netif_err(adapter, ifup, adapter->netdev,
@@ -341,6 +360,20 @@ static int ena_setup_rx_resources(struct ena_adapter *adapter,
return -ENOMEM;
}
+ size = sizeof(u16) * rx_ring->ring_size;
+ rx_ring->free_rx_ids = vzalloc_node(size, node);
+ if (!rx_ring->free_rx_ids) {
+ rx_ring->free_rx_ids = vzalloc(size);
+ if (!rx_ring->free_rx_ids) {
+ vfree(rx_ring->rx_buffer_info);
+ return -ENOMEM;
+ }
+ }
+
+ /* Req id ring for receiving RX pkts out of order */
+ for (i = 0; i < rx_ring->ring_size; i++)
+ rx_ring->free_rx_ids[i] = i;
+
/* Reset rx statistics */
memset(&rx_ring->rx_stats, 0x0, sizeof(rx_ring->rx_stats));
@@ -364,6 +397,9 @@ static void ena_free_rx_resources(struct ena_adapter *adapter,
vfree(rx_ring->rx_buffer_info);
rx_ring->rx_buffer_info = NULL;
+
+ vfree(rx_ring->free_rx_ids);
+ rx_ring->free_rx_ids = NULL;
}
/* ena_setup_all_rx_resources - allocate I/O Rx queues resources for all queues
@@ -469,15 +505,22 @@ static void ena_free_rx_page(struct ena_ring *rx_ring,
static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num)
{
- u16 next_to_use;
+ u16 next_to_use, req_id;
u32 i;
int rc;
next_to_use = rx_ring->next_to_use;
for (i = 0; i < num; i++) {
- struct ena_rx_buffer *rx_info =
- &rx_ring->rx_buffer_info[next_to_use];
+ struct ena_rx_buffer *rx_info;
+
+ req_id = rx_ring->free_rx_ids[next_to_use];
+ rc = validate_rx_req_id(rx_ring, req_id);
+ if (unlikely(rc < 0))
+ break;
+
+ rx_info = &rx_ring->rx_buffer_info[req_id];
+
rc = ena_alloc_rx_page(rx_ring, rx_info,
__GFP_COLD | GFP_ATOMIC | __GFP_COMP);
@@ -489,7 +532,7 @@ static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num)
}
rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
&rx_info->ena_buf,
- next_to_use);
+ req_id);
if (unlikely(rc)) {
netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
"failed to add buffer for rx queue %d\n",
@@ -676,6 +719,7 @@ static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)
u64_stats_update_end(&tx_ring->syncp);
/* Trigger device reset */
+ tx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
set_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags);
return -EFAULT;
}
@@ -799,7 +843,7 @@ static struct sk_buff *ena_alloc_skb(struct ena_ring *rx_ring, bool frags)
u64_stats_update_begin(&rx_ring->syncp);
rx_ring->rx_stats.skb_alloc_fail++;
u64_stats_update_end(&rx_ring->syncp);
- netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
+ netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
"Failed to allocate skb. frags: %d\n", frags);
return NULL;
}
@@ -813,12 +857,14 @@ static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring,
u16 *next_to_clean)
{
struct sk_buff *skb;
- struct ena_rx_buffer *rx_info =
- &rx_ring->rx_buffer_info[*next_to_clean];
- u32 len, buf = 0;
+ struct ena_rx_buffer *rx_info;
+ u16 len, req_id, buf = 0;
void *va;
- len = ena_bufs[0].len;
+ len = ena_bufs[buf].len;
+ req_id = ena_bufs[buf].req_id;
+ rx_info = &rx_ring->rx_buffer_info[req_id];
+
if (unlikely(!rx_info->page)) {
netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
"Page is NULL\n");
@@ -855,6 +901,7 @@ static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring,
skb_put(skb, len);
skb->protocol = eth_type_trans(skb, rx_ring->netdev);
+ rx_ring->free_rx_ids[*next_to_clean] = req_id;
*next_to_clean = ENA_RX_RING_IDX_ADD(*next_to_clean, descs,
rx_ring->ring_size);
return skb;
@@ -877,13 +924,18 @@ static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring,
skb->len, skb->data_len);
rx_info->page = NULL;
+
+ rx_ring->free_rx_ids[*next_to_clean] = req_id;
*next_to_clean =
ENA_RX_RING_IDX_NEXT(*next_to_clean,
rx_ring->ring_size);
if (likely(--descs == 0))
break;
- rx_info = &rx_ring->rx_buffer_info[*next_to_clean];
- len = ena_bufs[++buf].len;
+
+ buf++;
+ len = ena_bufs[buf].len;
+ req_id = ena_bufs[buf].req_id;
+ rx_info = &rx_ring->rx_buffer_info[req_id];
} while (1);
return skb;
@@ -918,7 +970,7 @@ static inline void ena_rx_checksum(struct ena_ring *rx_ring,
u64_stats_update_begin(&rx_ring->syncp);
rx_ring->rx_stats.bad_csum++;
u64_stats_update_end(&rx_ring->syncp);
- netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
+ netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
"RX IPv4 header checksum error\n");
return;
}
@@ -931,7 +983,7 @@ static inline void ena_rx_checksum(struct ena_ring *rx_ring,
u64_stats_update_begin(&rx_ring->syncp);
rx_ring->rx_stats.bad_csum++;
u64_stats_update_end(&rx_ring->syncp);
- netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
+ netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
"RX L4 checksum error\n");
skb->ip_summed = CHECKSUM_NONE;
return;
@@ -984,6 +1036,7 @@ static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,
int rc = 0;
int total_len = 0;
int rx_copybreak_pkt = 0;
+ int i;
netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
"%s qid %d\n", __func__, rx_ring->qid);
@@ -1013,9 +1066,13 @@ static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,
/* exit if we failed to retrieve a buffer */
if (unlikely(!skb)) {
- next_to_clean = ENA_RX_RING_IDX_ADD(next_to_clean,
- ena_rx_ctx.descs,
- rx_ring->ring_size);
+ for (i = 0; i < ena_rx_ctx.descs; i++) {
+ rx_ring->free_tx_ids[next_to_clean] =
+ rx_ring->ena_bufs[i].req_id;
+ next_to_clean =
+ ENA_RX_RING_IDX_NEXT(next_to_clean,
+ rx_ring->ring_size);
+ }
break;
}
@@ -1067,6 +1124,7 @@ static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,
u64_stats_update_end(&rx_ring->syncp);
/* Too many desc from the device. Trigger reset */
+ adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
return 0;
@@ -1091,6 +1149,26 @@ inline void ena_adjust_intr_moderation(struct ena_ring *rx_ring,
rx_ring->per_napi_bytes = 0;
}
+static inline void ena_unmask_interrupt(struct ena_ring *tx_ring,
+ struct ena_ring *rx_ring)
+{
+ struct ena_eth_io_intr_reg intr_reg;
+
+ /* Update intr register: rx intr delay,
+ * tx intr delay and interrupt unmask
+ */
+ ena_com_update_intr_reg(&intr_reg,
+ rx_ring->smoothed_interval,
+ tx_ring->smoothed_interval,
+ true);
+
+ /* It is a shared MSI-X.
+ * Tx and Rx CQ have pointer to it.
+ * So we use one of them to reach the intr reg
+ */
+ ena_com_unmask_intr(rx_ring->ena_com_io_cq, &intr_reg);
+}
+
static inline void ena_update_ring_numa_node(struct ena_ring *tx_ring,
struct ena_ring *rx_ring)
{
@@ -1121,7 +1199,6 @@ static int ena_io_poll(struct napi_struct *napi, int budget)
{
struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
struct ena_ring *tx_ring, *rx_ring;
- struct ena_eth_io_intr_reg intr_reg;
u32 tx_work_done;
u32 rx_work_done;
@@ -1157,28 +1234,15 @@ static int ena_io_poll(struct napi_struct *napi, int budget)
/* Update numa and unmask the interrupt only when schedule
* from the interrupt context (vs from sk_busy_loop)
*/
- napi_complete_done(napi, 0);
+ napi_complete_done(napi, rx_work_done);
if (atomic_cmpxchg(&ena_napi->unmask_interrupt, 1, 0)) {
/* Tx and Rx share the same interrupt vector */
if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev))
ena_adjust_intr_moderation(rx_ring, tx_ring);
- /* Update intr register: rx intr delay,
- * tx intr delay and interrupt unmask
- */
- ena_com_update_intr_reg(&intr_reg,
- rx_ring->smoothed_interval,
- tx_ring->smoothed_interval,
- true);
-
- /* It is a shared MSI-X.
- * Tx and Rx CQ have pointer to it.
- * So we use one of them to reach the intr reg
- */
- ena_com_unmask_intr(rx_ring->ena_com_io_cq, &intr_reg);
+ ena_unmask_interrupt(tx_ring, rx_ring);
}
-
ena_update_ring_numa_node(tx_ring, rx_ring);
ret = rx_work_done;
@@ -1215,17 +1279,20 @@ static irqreturn_t ena_intr_msix_io(int irq, void *data)
{
struct ena_napi *ena_napi = data;
- if (napi_schedule_prep(&ena_napi->napi)) {
- atomic_set(&ena_napi->unmask_interrupt, 1);
- __napi_schedule_irqoff(&ena_napi->napi);
- }
+ atomic_set(&ena_napi->unmask_interrupt, 1);
+ napi_schedule_irqoff(&ena_napi->napi);
return IRQ_HANDLED;
}
+/* Reserve a single MSI-X vector for management (admin + aenq).
+ * plus reserve one vector for each potential io queue.
+ * the number of potential io queues is the minimum of what the device
+ * supports and the number of vCPUs.
+ */
static int ena_enable_msix(struct ena_adapter *adapter, int num_queues)
{
- int i, msix_vecs, rc;
+ int msix_vecs, irq_cnt;
if (test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {
netif_err(adapter, probe, adapter->netdev,
@@ -1239,32 +1306,27 @@ static int ena_enable_msix(struct ena_adapter *adapter, int num_queues)
netif_dbg(adapter, probe, adapter->netdev,
"trying to enable MSI-X, vectors %d\n", msix_vecs);
- adapter->msix_entries = vzalloc(msix_vecs * sizeof(struct msix_entry));
-
- if (!adapter->msix_entries)
- return -ENOMEM;
+ irq_cnt = pci_alloc_irq_vectors(adapter->pdev, ENA_MIN_MSIX_VEC,
+ msix_vecs, PCI_IRQ_MSIX);
- for (i = 0; i < msix_vecs; i++)
- adapter->msix_entries[i].entry = i;
-
- rc = pci_enable_msix(adapter->pdev, adapter->msix_entries, msix_vecs);
- if (rc != 0) {
+ if (irq_cnt < 0) {
netif_err(adapter, probe, adapter->netdev,
- "Failed to enable MSI-X, vectors %d rc %d\n",
- msix_vecs, rc);
+ "Failed to enable MSI-X. irq_cnt %d\n", irq_cnt);
return -ENOSPC;
}
- netif_dbg(adapter, probe, adapter->netdev, "enable MSI-X, vectors %d\n",
- msix_vecs);
-
- if (msix_vecs >= 1) {
- if (ena_init_rx_cpu_rmap(adapter))
- netif_warn(adapter, probe, adapter->netdev,
- "Failed to map IRQs to CPUs\n");
+ if (irq_cnt != msix_vecs) {
+ netif_notice(adapter, probe, adapter->netdev,
+ "enable only %d MSI-X (out of %d), reduce the number of queues\n",
+ irq_cnt, msix_vecs);
+ adapter->num_queues = irq_cnt - ENA_ADMIN_MSIX_VEC;
}
- adapter->msix_vecs = msix_vecs;
+ if (ena_init_rx_cpu_rmap(adapter))
+ netif_warn(adapter, probe, adapter->netdev,
+ "Failed to map IRQs to CPUs\n");
+
+ adapter->msix_vecs = irq_cnt;
set_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags);
return 0;
@@ -1281,7 +1343,7 @@ static void ena_setup_mgmnt_intr(struct ena_adapter *adapter)
ena_intr_msix_mgmnt;
adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;
adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =
- adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector;
+ pci_irq_vector(adapter->pdev, ENA_MGMNT_IRQ_IDX);
cpu = cpumask_first(cpu_online_mask);
adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].cpu = cpu;
cpumask_set_cpu(cpu,
@@ -1304,7 +1366,7 @@ static void ena_setup_io_intr(struct ena_adapter *adapter)
adapter->irq_tbl[irq_idx].handler = ena_intr_msix_io;
adapter->irq_tbl[irq_idx].data = &adapter->ena_napi[i];
adapter->irq_tbl[irq_idx].vector =
- adapter->msix_entries[irq_idx].vector;
+ pci_irq_vector(adapter->pdev, irq_idx);
adapter->irq_tbl[irq_idx].cpu = cpu;
cpumask_set_cpu(cpu,
@@ -1405,11 +1467,7 @@ static void ena_free_io_irq(struct ena_adapter *adapter)
static void ena_disable_msix(struct ena_adapter *adapter)
{
if (test_and_clear_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags))
- pci_disable_msix(adapter->pdev);
-
- if (adapter->msix_entries)
- vfree(adapter->msix_entries);
- adapter->msix_entries = NULL;
+ pci_free_irq_vectors(adapter->pdev);
}
static void ena_disable_io_intr_sync(struct ena_adapter *adapter)
@@ -1482,7 +1540,7 @@ static int ena_rss_configure(struct ena_adapter *adapter)
/* In case the RSS table wasn't initialized by probe */
if (!ena_dev->rss.tbl_log_size) {
rc = ena_rss_init_default(adapter);
- if (rc && (rc != -EPERM)) {
+ if (rc && (rc != -EOPNOTSUPP)) {
netif_err(adapter, ifup, adapter->netdev,
"Failed to init RSS rc: %d\n", rc);
return rc;
@@ -1491,17 +1549,17 @@ static int ena_rss_configure(struct ena_adapter *adapter)
/* Set indirect table */
rc = ena_com_indirect_table_set(ena_dev);
- if (unlikely(rc && rc != -EPERM))
+ if (unlikely(rc && rc != -EOPNOTSUPP))
return rc;
/* Configure hash function (if supported) */
rc = ena_com_set_hash_function(ena_dev);
- if (unlikely(rc && (rc != -EPERM)))
+ if (unlikely(rc && (rc != -EOPNOTSUPP)))
return rc;
/* Configure hash inputs (if supported) */
rc = ena_com_set_hash_ctrl(ena_dev);
- if (unlikely(rc && (rc != -EPERM)))
+ if (unlikely(rc && (rc != -EOPNOTSUPP)))
return rc;
return 0;
@@ -1528,6 +1586,11 @@ static int ena_up_complete(struct ena_adapter *adapter)
ena_napi_enable_all(adapter);
+ /* Enable completion queues interrupt */
+ for (i = 0; i < adapter->num_queues; i++)
+ ena_unmask_interrupt(&adapter->tx_ring[i],
+ &adapter->rx_ring[i]);
+
/* schedule napi in case we had pending packets
* from the last time we disable napi
*/
@@ -1575,6 +1638,7 @@ static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid)
"Failed to get TX queue handlers. TX queue num %d rc: %d\n",
qid, rc);
ena_com_destroy_io_queue(ena_dev, ena_qid);
+ return rc;
}
ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node);
@@ -1639,6 +1703,7 @@ static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid)
"Failed to get RX queue handlers. RX queue num %d rc: %d\n",
qid, rc);
ena_com_destroy_io_queue(ena_dev, ena_qid);
+ return rc;
}
ena_com_update_numa_node(rx_ring->ena_com_io_cq, ctx.numa_node);
@@ -1749,7 +1814,7 @@ static void ena_down(struct ena_adapter *adapter)
if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) {
int rc;
- rc = ena_com_dev_reset(adapter->ena_dev);
+ rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
if (rc)
dev_err(&adapter->pdev->dev, "Device reset failed\n");
}
@@ -1913,7 +1978,7 @@ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
struct netdev_queue *txq;
struct ena_com_buf *ena_buf;
void *push_hdr;
- u32 len, last_frag;
+ u32 len, pkt_len, last_frag;
u16 next_to_use;
u16 req_id;
u16 push_len;
@@ -1934,6 +1999,7 @@ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
skb_tx_timestamp(skb);
len = skb_headlen(skb);
+ pkt_len = skb->len;
next_to_use = tx_ring->next_to_use;
req_id = tx_ring->free_tx_ids[next_to_use];
@@ -1946,13 +2012,13 @@ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
/* prepared the push buffer */
- push_len = min_t(u32, len, tx_ring->tx_max_header_size);
+ push_len = min_t(u32, pkt_len, tx_ring->tx_max_header_size);
header_len = push_len;
push_hdr = skb->data;
} else {
+ push_hdr = NULL;
push_len = 0;
header_len = min_t(u32, len, tx_ring->tx_max_header_size);
- push_hdr = NULL;
}
netif_dbg(adapter, tx_queued, dev,
@@ -2171,7 +2237,7 @@ static void ena_config_host_info(struct ena_com_dev *ena_dev)
rc = ena_com_set_host_attributes(ena_dev);
if (rc) {
- if (rc == -EPERM)
+ if (rc == -EOPNOTSUPP)
pr_warn("Cannot set host attributes\n");
else
pr_err("Cannot set host attributes\n");
@@ -2208,7 +2274,7 @@ static void ena_config_debug_area(struct ena_adapter *adapter)
rc = ena_com_set_host_attributes(adapter->ena_dev);
if (rc) {
- if (rc == -EPERM)
+ if (rc == -EOPNOTSUPP)
netif_warn(adapter, drv, adapter->netdev,
"Cannot set host attributes\n");
else
@@ -2279,8 +2345,7 @@ static struct rtnl_link_stats64 *ena_get_stats64(struct net_device *netdev,
stats->rx_errors = 0;
stats->tx_errors = 0;
-
- return stats;
+ return stats;
}
static const struct net_device_ops ena_netdev_ops = {
@@ -2382,7 +2447,7 @@ static int ena_device_init(struct ena_com_dev *ena_dev, struct pci_dev *pdev,
readless_supported = !(pdev->revision & ENA_MMIO_DISABLE_REG_READ);
ena_com_set_mmio_read_mode(ena_dev, readless_supported);
- rc = ena_com_dev_reset(ena_dev);
+ rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);
if (rc) {
dev_err(dev, "Can not reset device\n");
goto err_mmio_read_less;
@@ -2542,6 +2607,7 @@ static void ena_fw_reset_device(struct work_struct *work)
ena_com_mmio_reg_read_request_destroy(ena_dev);
+ adapter->reset_reason = ENA_REGS_RESET_NORMAL;
clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
/* Finish with the destroy part. Start the init part */
@@ -2595,13 +2661,47 @@ static void ena_fw_reset_device(struct work_struct *work)
"Reset attempt failed. Can not reset the device\n");
}
-static void check_for_missing_tx_completions(struct ena_adapter *adapter)
+static int check_missing_comp_in_queue(struct ena_adapter *adapter,
+ struct ena_ring *tx_ring)
{
struct ena_tx_buffer *tx_buf;
unsigned long last_jiffies;
+ u32 missed_tx = 0;
+ int i;
+
+ for (i = 0; i < tx_ring->ring_size; i++) {
+ tx_buf = &tx_ring->tx_buffer_info[i];
+ last_jiffies = tx_buf->last_jiffies;
+ if (unlikely(last_jiffies &&
+ time_is_before_jiffies(last_jiffies + adapter->missing_tx_completion_to))) {
+ if (!tx_buf->print_once)
+ netif_notice(adapter, tx_err, adapter->netdev,
+ "Found a Tx that wasn't completed on time, qid %d, index %d.\n",
+ tx_ring->qid, i);
+
+ tx_buf->print_once = 1;
+ missed_tx++;
+
+ if (unlikely(missed_tx > adapter->missing_tx_completion_threshold)) {
+ netif_err(adapter, tx_err, adapter->netdev,
+ "The number of lost tx completions is above the threshold (%d > %d). Reset the device\n",
+ missed_tx,
+ adapter->missing_tx_completion_threshold);
+ adapter->reset_reason =
+ ENA_REGS_RESET_MISS_TX_CMPL;
+ set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
+ return -EIO;
+ }
+ }
+ }
+
+ return 0;
+}
+
+static void check_for_missing_tx_completions(struct ena_adapter *adapter)
+{
struct ena_ring *tx_ring;
- int i, j, budget;
- u32 missed_tx;
+ int i, budget, rc;
/* Make sure the driver doesn't turn the device in other process */
smp_rmb();
@@ -2620,29 +2720,9 @@ static void check_for_missing_tx_completions(struct ena_adapter *adapter)
for (i = adapter->last_monitored_tx_qid; i < adapter->num_queues; i++) {
tx_ring = &adapter->tx_ring[i];
- missed_tx = 0;
-
- for (j = 0; j < tx_ring->ring_size; j++) {
- tx_buf = &tx_ring->tx_buffer_info[j];
- last_jiffies = tx_buf->last_jiffies;
- if (unlikely(last_jiffies && time_is_before_jiffies(last_jiffies + adapter->missing_tx_completion_to))) {
- if (!tx_buf->print_once)
- netif_notice(adapter, tx_err, adapter->netdev,
- "Found a Tx that wasn't completed on time, qid %d, index %d.\n",
- tx_ring->qid, j);
-
- tx_buf->print_once = 1;
- missed_tx++;
-
- if (unlikely(missed_tx > adapter->missing_tx_completion_threshold)) {
- netif_err(adapter, tx_err, adapter->netdev,
- "The number of lost tx completion is above the threshold (%d > %d). Reset the device\n",
- missed_tx, adapter->missing_tx_completion_threshold);
- set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
- return;
- }
- }
- }
+ rc = check_missing_comp_in_queue(adapter, tx_ring);
+ if (unlikely(rc))
+ return;
budget--;
if (!budget)
@@ -2652,6 +2732,58 @@ static void check_for_missing_tx_completions(struct ena_adapter *adapter)
adapter->last_monitored_tx_qid = i % adapter->num_queues;
}
+/* trigger napi schedule after 2 consecutive detections */
+#define EMPTY_RX_REFILL 2
+/* For the rare case where the device runs out of Rx descriptors and the
+ * napi handler failed to refill new Rx descriptors (due to a lack of memory
+ * for example).
+ * This case will lead to a deadlock:
+ * The device won't send interrupts since all the new Rx packets will be dropped
+ * The napi handler won't allocate new Rx descriptors so the device will be
+ * able to send new packets.
+ *
+ * This scenario can happen when the kernel's vm.min_free_kbytes is too small.
+ * It is recommended to have at least 512MB, with a minimum of 128MB for
+ * constrained environment).
+ *
+ * When such a situation is detected - Reschedule napi
+ */
+static void check_for_empty_rx_ring(struct ena_adapter *adapter)
+{
+ struct ena_ring *rx_ring;
+ int i, refill_required;
+
+ if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
+ return;
+
+ if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
+ return;
+
+ for (i = 0; i < adapter->num_queues; i++) {
+ rx_ring = &adapter->rx_ring[i];
+
+ refill_required =
+ ena_com_sq_empty_space(rx_ring->ena_com_io_sq);
+ if (unlikely(refill_required == (rx_ring->ring_size - 1))) {
+ rx_ring->empty_rx_queue++;
+
+ if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) {
+ u64_stats_update_begin(&rx_ring->syncp);
+ rx_ring->rx_stats.empty_rx_ring++;
+ u64_stats_update_end(&rx_ring->syncp);
+
+ netif_err(adapter, drv, adapter->netdev,
+ "trigger refill for ring %d\n", i);
+
+ napi_schedule(rx_ring->napi);
+ rx_ring->empty_rx_queue = 0;
+ }
+ } else {
+ rx_ring->empty_rx_queue = 0;
+ }
+ }
+}
+
/* Check for keep alive expiration */
static void check_for_missing_keep_alive(struct ena_adapter *adapter)
{
@@ -2671,6 +2803,7 @@ static void check_for_missing_keep_alive(struct ena_adapter *adapter)
u64_stats_update_begin(&adapter->syncp);
adapter->dev_stats.wd_expired++;
u64_stats_update_end(&adapter->syncp);
+ adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO;
set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
}
}
@@ -2683,6 +2816,7 @@ static void check_for_admin_com_state(struct ena_adapter *adapter)
u64_stats_update_begin(&adapter->syncp);
adapter->dev_stats.admin_q_pause++;
u64_stats_update_end(&adapter->syncp);
+ adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO;
set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
}
}
@@ -2747,6 +2881,8 @@ static void ena_timer_service(unsigned long data)
check_for_missing_tx_completions(adapter);
+ check_for_empty_rx_ring(adapter);
+
if (debug_area)
ena_dump_stats_to_buf(adapter, debug_area);
@@ -2852,15 +2988,7 @@ static void ena_set_dev_offloads(struct ena_com_dev_get_features_ctx *feat,
NETIF_F_RXHASH |
NETIF_F_HIGHDMA;
-#ifdef HAVE_RHEL6_NET_DEVICE_OPS_EXT
- do {
- u32 hw_features = get_netdev_hw_features(netdev);
- hw_features |= netdev->features;
- set_netdev_hw_features(netdev, hw_features);
- } while (0);
-#else
netdev->hw_features |= netdev->features;
-#endif
netdev->vlan_features |= netdev->features;
}
@@ -2901,7 +3029,7 @@ static int ena_rss_init_default(struct ena_adapter *adapter)
val = ethtool_rxfh_indir_default(i, adapter->num_queues);
rc = ena_com_indirect_table_fill_entry(ena_dev, i,
ENA_IO_RXQ_IDX(val));
- if (unlikely(rc && (rc != -EPERM))) {
+ if (unlikely(rc && (rc != -EOPNOTSUPP))) {
dev_err(dev, "Cannot fill indirect table\n");
goto err_fill_indir;
}
@@ -2909,13 +3037,13 @@ static int ena_rss_init_default(struct ena_adapter *adapter)
rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL,
ENA_HASH_KEY_SIZE, 0xFFFFFFFF);
- if (unlikely(rc && (rc != -EPERM))) {
+ if (unlikely(rc && (rc != -EOPNOTSUPP))) {
dev_err(dev, "Cannot fill hash function\n");
goto err_fill_indir;
}
rc = ena_com_set_default_hash_ctrl(ena_dev);
- if (unlikely(rc && (rc != -EPERM))) {
+ if (unlikely(rc && (rc != -EOPNOTSUPP))) {
dev_err(dev, "Cannot fill hash control\n");
goto err_fill_indir;
}
@@ -2933,6 +3061,11 @@ static void ena_release_bars(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
{
int release_bars;
+ if (ena_dev->mem_bar)
+ devm_iounmap(&pdev->dev, ena_dev->mem_bar);
+
+ devm_iounmap(&pdev->dev, ena_dev->reg_bar);
+
release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
pci_release_selected_regions(pdev, release_bars);
}
@@ -3020,8 +3153,9 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
goto err_free_ena_dev;
}
- ena_dev->reg_bar = ioremap(pci_resource_start(pdev, ENA_REG_BAR),
- pci_resource_len(pdev, ENA_REG_BAR));
+ ena_dev->reg_bar = devm_ioremap(&pdev->dev,
+ pci_resource_start(pdev, ENA_REG_BAR),
+ pci_resource_len(pdev, ENA_REG_BAR));
if (!ena_dev->reg_bar) {
dev_err(&pdev->dev, "failed to remap regs bar\n");
rc = -EFAULT;
@@ -3041,8 +3175,9 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
ena_set_push_mode(pdev, ena_dev, &get_feat_ctx);
if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
- ena_dev->mem_bar = ioremap_wc(pci_resource_start(pdev, ENA_MEM_BAR),
- pci_resource_len(pdev, ENA_MEM_BAR));
+ ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev,
+ pci_resource_start(pdev, ENA_MEM_BAR),
+ pci_resource_len(pdev, ENA_MEM_BAR));
if (!ena_dev->mem_bar) {
rc = -EFAULT;
goto err_device_destroy;
@@ -3084,6 +3219,7 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
ena_set_conf_feat_params(adapter, &get_feat_ctx);
adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
+ adapter->reset_reason = ENA_REGS_RESET_NORMAL;
adapter->tx_ring_size = queue_size;
adapter->rx_ring_size = queue_size;
@@ -3122,7 +3258,7 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
goto err_worker_destroy;
}
rc = ena_rss_init_default(adapter);
- if (rc && (rc != -EPERM)) {
+ if (rc && (rc != -EOPNOTSUPP)) {
dev_err(&pdev->dev, "Cannot init RSS rc: %d\n", rc);
goto err_free_msix;
}
@@ -3150,12 +3286,9 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
ena_update_hints(adapter, &get_feat_ctx.hw_hints);
- init_timer(&adapter->timer_service);
- adapter->timer_service.expires = round_jiffies(jiffies + HZ);
- adapter->timer_service.function = ena_timer_service;
- adapter->timer_service.data = (unsigned long)adapter;
-
- add_timer(&adapter->timer_service);
+ setup_timer(&adapter->timer_service, ena_timer_service,
+ (unsigned long)adapter);
+ mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
dev_info(&pdev->dev, "%s found at mem %lx, mac addr %pM Queues %d\n",
DEVICE_NAME, (long)pci_resource_start(pdev, 0),
@@ -3171,7 +3304,7 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
ena_com_delete_debug_area(ena_dev);
ena_com_rss_destroy(ena_dev);
err_free_msix:
- ena_com_dev_reset(ena_dev);
+ ena_com_dev_reset(ena_dev, ENA_REGS_RESET_INIT_ERR);
ena_free_mgmnt_irq(adapter);
ena_disable_msix(adapter);
err_worker_destroy:
@@ -3187,7 +3320,6 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
err_free_region:
ena_release_bars(ena_dev, pdev);
err_free_ena_dev:
- pci_set_drvdata(pdev, NULL);
vfree(ena_dev);
err_disable_device:
pci_disable_device(pdev);
@@ -3234,12 +3366,6 @@ static void ena_remove(struct pci_dev *pdev)
struct ena_com_dev *ena_dev;
struct net_device *netdev;
- if (!adapter)
- /* This device didn't load properly and it's resources
- * already released, nothing to do
- */
- return;
-
ena_dev = adapter->ena_dev;
netdev = adapter->netdev;
@@ -3261,7 +3387,7 @@ static void ena_remove(struct pci_dev *pdev)
/* Reset the device only if the device is running. */
if (test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
- ena_com_dev_reset(ena_dev);
+ ena_com_dev_reset(ena_dev, adapter->reset_reason);
ena_free_mgmnt_irq(adapter);
diff --git a/drivers/amazon/net/ena/ena_netdev.h b/drivers/amazon/net/ena/ena_netdev.h
index 6b4fec4..d99105c 100644
--- a/drivers/amazon/net/ena/ena_netdev.h
+++ b/drivers/amazon/net/ena/ena_netdev.h
@@ -45,21 +45,24 @@
#include "ena_eth_com.h"
#define DRV_MODULE_VER_MAJOR 1
-#define DRV_MODULE_VER_MINOR 1
-#define DRV_MODULE_VER_SUBMINOR 3
+#define DRV_MODULE_VER_MINOR 2
+#define DRV_MODULE_VER_SUBMINOR 0
#define DRV_MODULE_NAME "ena"
#ifndef DRV_MODULE_VERSION
#define DRV_MODULE_VERSION \
__stringify(DRV_MODULE_VER_MAJOR) "." \
__stringify(DRV_MODULE_VER_MINOR) "." \
- __stringify(DRV_MODULE_VER_SUBMINOR)
+ __stringify(DRV_MODULE_VER_SUBMINOR) "u"
#endif
#define DEVICE_NAME "Elastic Network Adapter (ENA)"
/* 1 for AENQ + ADMIN */
-#define ENA_MAX_MSIX_VEC(io_queues) (1 + (io_queues))
+#define ENA_ADMIN_MSIX_VEC 1
+#define ENA_MAX_MSIX_VEC(io_queues) (ENA_ADMIN_MSIX_VEC + (io_queues))
+
+#define ENA_MIN_MSIX_VEC 2
#define ENA_REG_BAR 0
#define ENA_MEM_BAR 2
@@ -196,11 +199,19 @@ struct ena_stats_rx {
u64 dma_mapping_err;
u64 bad_desc_num;
u64 rx_copybreak_pkt;
+ u64 bad_req_id;
+ u64 empty_rx_ring;
};
struct ena_ring {
- /* Holds the empty requests for TX out of order completions */
- u16 *free_tx_ids;
+ union {
+ /* Holds the empty requests for TX/RX
+ * out of order completions
+ */
+ u16 *free_tx_ids;
+ u16 *free_rx_ids;
+ };
+
union {
struct ena_tx_buffer *tx_buffer_info;
struct ena_rx_buffer *rx_buffer_info;
@@ -243,6 +254,7 @@ struct ena_ring {
struct ena_stats_tx tx_stats;
struct ena_stats_rx rx_stats;
};
+ int empty_rx_queue;
} ____cacheline_aligned;
struct ena_stats_dev {
@@ -279,7 +291,6 @@ struct ena_adapter {
int num_queues;
- struct msix_entry *msix_entries;
int msix_vecs;
u32 missing_tx_completion_threshold;
@@ -329,6 +340,8 @@ struct ena_adapter {
/* last queue index that was checked for uncompleted tx packets */
u32 last_monitored_tx_qid;
+
+ enum ena_regs_reset_reason_types reset_reason;
};
void ena_set_ethtool_ops(struct net_device *netdev);
diff --git a/drivers/amazon/net/ena/ena_regs_defs.h b/drivers/amazon/net/ena/ena_regs_defs.h
index c3891c5..9aec43c 100644
--- a/drivers/amazon/net/ena/ena_regs_defs.h
+++ b/drivers/amazon/net/ena/ena_regs_defs.h
@@ -32,6 +32,36 @@
#ifndef _ENA_REGS_H_
#define _ENA_REGS_H_
+enum ena_regs_reset_reason_types {
+ ENA_REGS_RESET_NORMAL = 0,
+
+ ENA_REGS_RESET_KEEP_ALIVE_TO = 1,
+
+ ENA_REGS_RESET_ADMIN_TO = 2,
+
+ ENA_REGS_RESET_MISS_TX_CMPL = 3,
+
+ ENA_REGS_RESET_INV_RX_REQ_ID = 4,
+
+ ENA_REGS_RESET_INV_TX_REQ_ID = 5,
+
+ ENA_REGS_RESET_TOO_MANY_RX_DESCS = 6,
+
+ ENA_REGS_RESET_INIT_ERR = 7,
+
+ ENA_REGS_RESET_DRIVER_INVALID_STATE = 8,
+
+ ENA_REGS_RESET_OS_TRIGGER = 9,
+
+ ENA_REGS_RESET_OS_NETDEV_WD = 10,
+
+ ENA_REGS_RESET_SHUTDOWN = 11,
+
+ ENA_REGS_RESET_USER_TRIGGER = 12,
+
+ ENA_REGS_RESET_GENERIC = 13,
+};
+
/* ena_registers offsets */
#define ENA_REGS_VERSION_OFF 0x0
#define ENA_REGS_CONTROLLER_VERSION_OFF 0x4
@@ -104,6 +134,8 @@
#define ENA_REGS_DEV_CTL_QUIESCENT_MASK 0x4
#define ENA_REGS_DEV_CTL_IO_RESUME_SHIFT 3
#define ENA_REGS_DEV_CTL_IO_RESUME_MASK 0x8
+#define ENA_REGS_DEV_CTL_RESET_REASON_SHIFT 28
+#define ENA_REGS_DEV_CTL_RESET_REASON_MASK 0xf0000000
/* dev_sts register */
#define ENA_REGS_DEV_STS_READY_MASK 0x1
--
2.7.5