// SPDX-License-Identifier: GPL-2.0
/* ldc.c: Logical Domain Channel link-layer protocol driver.
*
* Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
*/
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/delay.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/scatterlist.h>
#include <linux/interrupt.h>
#include <linux/list.h>
#include <linux/init.h>
#include <linux/bitmap.h>
#include <asm/iommu-common.h>
#include <asm/hypervisor.h>
#include <asm/iommu.h>
#include <asm/page.h>
#include <asm/ldc.h>
#include <asm/mdesc.h>
#define DRV_MODULE_NAME "ldc"
#define PFX DRV_MODULE_NAME ": "
#define DRV_MODULE_VERSION "1.1"
#define DRV_MODULE_RELDATE "July 22, 2008"
#define COOKIE_PGSZ_CODE 0xf000000000000000ULL
#define COOKIE_PGSZ_CODE_SHIFT 60ULL
static char version[] =
DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
/* Packet header layout for unreliable and reliable mode frames.
* When in RAW mode, packets are simply straight 64-byte payloads
* with no headers.
*/
struct ldc_packet {
u8 type;
#define LDC_CTRL 0x01
#define LDC_DATA 0x02
#define LDC_ERR 0x10
u8 stype;
#define LDC_INFO 0x01
#define LDC_ACK 0x02
#define LDC_NACK 0x04
u8 ctrl;
#define LDC_VERS 0x01 /* Link Version */
#define LDC_RTS 0x02 /* Request To Send */
#define LDC_RTR 0x03 /* Ready To Receive */
#define LDC_RDX 0x04 /* Ready for Data eXchange */
#define LDC_CTRL_MSK 0x0f
u8 env;
#define LDC_LEN 0x3f
#define LDC_FRAG_MASK 0xc0
#define LDC_START 0x40
#define LDC_STOP 0x80
u32 seqid;
union {
u8 u_data[LDC_PACKET_SIZE - 8];
struct {
u32 pad;
u32 ackid;
u8 r_data[LDC_PACKET_SIZE - 8 - 8];
} r;
} u;
};
struct ldc_version {
u16 major;
u16 minor;
};
/* Ordered from largest major to lowest. */
static struct ldc_version ver_arr[] = {
{ .major = 1, .minor = 0 },
};
#define LDC_DEFAULT_MTU (4 * LDC_PACKET_SIZE)
#define LDC_DEFAULT_NUM_ENTRIES (PAGE_SIZE / LDC_PACKET_SIZE)
struct ldc_channel;
struct ldc_mode_ops {
int (*write)(struct ldc_channel *, const void *, unsigned int);
int (*read)(struct ldc_channel *, void *, unsigned int);
};
static const struct ldc_mode_ops raw_ops;
static const struct ldc_mode_ops nonraw_ops;
static const struct ldc_mode_ops stream_ops;
int ldom_domaining_enabled;
struct ldc_iommu {
/* Protects ldc_unmap. */
spinlock_t lock;
struct ldc_mtable_entry *page_table;
struct iommu_map_table iommu_map_table;
};
struct ldc_channel {
/* Protects all operations that depend upon channel state. */
spinlock_t lock;
unsigned long id;
u8 *mssbuf;
u32 mssbuf_len;
u32 mssbuf_off;
struct ldc_packet *tx_base;
unsigned long tx_head;
unsigned long tx_tail;
unsigned long tx_num_entries;
unsigned long tx_ra;
unsigned long tx_acked;
struct ldc_packet *rx_base;
unsigned long rx_head;
unsigned long rx_tail;
unsigned long rx_num_entries;
unsigned long rx_ra;
u32 rcv_nxt;
u32 snd_nxt;
unsigned long chan_state;
struct ldc_channel_config cfg;
void *event_arg;
const struct ldc_mode_ops *mops;
struct ldc_iommu iommu;
struct ldc_version ver;
u8 hs_state;
#define LDC_HS_CLOSED 0x00
#define LDC_HS_OPEN 0x01
#define LDC_HS_GOTVERS 0x02
#define LDC_HS_SENTRTR 0x03
#define LDC_HS_GOTRTR 0x04
#define LDC_HS_COMPLETE 0x10
u8 flags;
#define LDC_FLAG_ALLOCED_QUEUES 0x01
#define LDC_FLAG_REGISTERED_QUEUES 0x02
#define LDC_FLAG_REGISTERED_IRQS 0x04
#define LDC_FLAG_RESET 0x10
u8 mss;
u8 state;
#define LDC_IRQ_NAME_MAX 32
char rx_irq_name[LDC_IRQ_NAME_MAX];
char tx_irq_name[LDC_IRQ_NAME_MAX];
struct hlist_head mh_list;
struct hlist_node list;
};
#define ldcdbg(TYPE, f, a...) \
do { if (lp->cfg.debug & LDC_DEBUG_##TYPE) \
printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \
} while (0)
#define LDC_ABORT(lp) ldc_abort((lp), __func__)
static const char *state_to_str(u8 state)
{
switch (state) {
case LDC_STATE_INVALID:
return "INVALID";
case LDC_STATE_INIT:
return "INIT";
case LDC_STATE_BOUND:
return "BOUND";
case LDC_STATE_READY:
return "READY";
case LDC_STATE_CONNECTED:
return "CONNECTED";
default:
return "<UNKNOWN>";
}
}
static unsigned long __advance(unsigned long off, unsigned long num_entries)
{
off += LDC_PACKET_SIZE;
if (off == (num_entries * LDC_PACKET_SIZE))
off = 0;
return off;
}
static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off)
{
return __advance(off, lp->rx_num_entries);
}
static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off)
{
return __advance(off, lp->tx_num_entries);
}
static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp,
unsigned long *new_tail)
{
struct ldc_packet *p;
unsigned long t;
t = tx_advance(lp, lp->tx_tail);
if (t == lp->tx_head)
return NULL;
*new_tail = t;
p = lp->tx_base;
return p + (lp->tx_tail / LDC_PACKET_SIZE);
}
/* When we are in reliable or stream mode, have to track the next packet
* we haven't gotten an ACK for in the TX queue using tx_acked. We have
* to be careful not to stomp over the queue past that point. During
* the handshake, we don't have TX data packets pending in the queue
* and that's why handshake_get_tx_packet() need not be mindful of
* lp->tx_acked.
*/
static unsigned long head_for_data(struct ldc_channel *lp)
{
if (lp->cfg.mode == LDC_MODE_STREAM)
return lp->tx_acked;
return lp->tx_head;
}
static int tx_has_space_for(struct ldc_channel *lp, unsigned int size)
{
unsigned long limit, tail, new_tail, diff;
unsigned int mss;
limit = head_for_data(lp);
tail = lp->tx_tail;
new_tail = tx_advance(lp, tail);
if (new_tail == limit)
return 0;
if (limit > new_tail)
diff = limit - new_tail;
else
diff = (limit +
((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail));
diff /= LDC_PACKET_SIZE;
mss = lp->mss;
if (diff * mss < size)
return 0;
return 1;
}
static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp,
unsigned long *new_tail)
{
struct ldc_packet *p;
unsigned long h, t;
h = head_for_data(lp);
t = tx_advance(lp, lp->tx_tail);
if (t == h)
return NULL;
*new_tail = t;
p = lp->tx_base;
return p + (lp->tx_tail / LDC_PACKET_SIZE);
}
static int set_tx_tail(struct ldc_channel *lp, unsigned long tail)
{
unsigned long orig_tail = lp->tx_tail;
int limit = 1000;
lp->tx_tail = tail;
while (limit-- > 0) {
unsigned long err;
err = sun4v_ldc_tx_set_qtail(lp->id, tail);
if (!err)
return 0;
if (err != HV_EWOULDBLOCK) {
lp->tx_tail = orig_tail;
return -EINVAL;
}
udelay(1);
}
lp->tx_tail = orig_tail;
return -EBUSY;
}
/* This just updates the head value in the hypervisor using
* a polling loop with a timeout. The caller takes care of
* upating software state representing the head change, if any.
*/
static int __set_rx_head(struct ldc_channel *lp, unsigned long head)
{
int limit = 1000;
while (limit-- > 0) {
unsigned long err;
err = sun4v_ldc_rx_set_qhead(lp->id, head);
if (!err)
return 0;
if (err != HV_EWOULDBLOCK)
return -EINVAL;
udelay(1);
}
return -EBUSY;
}
static int send_tx_packet(struct ldc_channel *lp,
struct ldc_packet *p,
unsigned long new_tail)
{
BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE)));
return set_tx_tail(lp, new_tail);
}
static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp,
u8 stype, u8 ctrl,
void *data, int dlen,
unsigned long *new_tail)
{
struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail);
if (p) {
memset(p, 0, sizeof(*p));
p->type = LDC_CTRL;
p->stype = stype;
p->ctrl = ctrl;
if (data)
memcpy(p->u.u_data, data, dlen);
}
return p;
}
static int start_handshake(struct ldc_channel *lp)
{
struct ldc_packet *p;
struct ldc_version *ver;
unsigned long new_tail;
ver = &ver_arr[0];
ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n",
ver->major, ver->minor);
p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
ver, sizeof(*ver), &new_tail);
if (p) {
int err = send_tx_packet(lp, p, new_tail);
if (!err)
lp->flags &= ~LDC_FLAG_RESET;
return err;
}
return -EBUSY;
}
static int send_version_nack(struct ldc_channel *lp,
u16 major, u16 minor)
{
struct ldc_packet *p;
struct ldc_version ver;
unsigned long new_tail;
ver.major = major;
ver.minor = minor;
p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS,
&ver, sizeof(ver), &new_tail);
if (p) {
ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n",
ver.major, ver.minor);
return send_tx_packet(lp, p, new_tail);
}
return -EBUSY;
}
static int send_version_ack(struct ldc_channel *lp,
struct ldc_version *vp)
{
struct ldc_packet *p;
unsigned long new_tail;
p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS,
vp, sizeof(*vp), &new_tail);
if (p) {
ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n",
vp->major, vp->minor);
return send_tx_packet(lp, p, new_tail);
}
return -EBUSY;
}
static int send_rts(struct ldc_channel *lp)
{
struct ldc_packet *p;
unsigned long new_tail;
p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0,
&new_tail);
if (p) {
p->env = lp->cfg.mode;
p->seqid = 0;
lp->rcv_nxt = 0;
ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n",
p->env, p->seqid);
return send_tx_packet(lp, p, new_tail);
}
return -EBUSY;
}
static int send_rtr(struct ldc_channel *lp)
{
struct ldc_packet *p;
unsigned long new_tail;
p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0,
&new_tail);
if (p) {
p->env = lp->cfg.mode;
p->seqid = 0;
ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n",
p->env, p->seqid);
return send_tx_packet(lp, p, new_tail);
}
return -EBUSY;
}
static int send_rdx(struct ldc_channel *lp)
{
struct ldc_packet *p;
unsigned long new_tail;
p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0,
&new_tail);
if (p) {
p->env = 0;
p->seqid = ++lp->snd_nxt;
p->u.r.ackid = lp->rcv_nxt;
ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n",
p->env, p->seqid, p->u.r.ackid);
return send_tx_packet(lp, p, new_tail);
}
return -EBUSY;
}
static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt)
{
struct ldc_packet *p;
unsigned long new_tail;
int err;
p = data_get_tx_packet(lp, &new_tail);
if (!p)
return -EBUSY;
memset(p, 0, sizeof(*p));
p->type = data_pkt->type;
p->stype = LDC_NACK;
p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK;
p->seqid = lp->snd_nxt + 1;
p->u.r.ackid = lp->rcv_nxt;
ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n",
p->type, p->ctrl, p->seqid, p->u.r.ackid);
err = send_tx_packet(lp, p, new_tail);
if (!err)
lp->snd_nxt++;
return err;
}
static int ldc_abort(struct ldc_channel *lp, const char *msg)
{
unsigned long hv_err;
ldcdbg(STATE, "ABORT[%s]\n", msg);
ldc_print(lp);
/* We report but do not act upon the hypervisor errors because
* there really isn't much we can do if they fail at this point.
*/
hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
if (hv_err)
printk(KERN_ERR PFX "ldc_abort: "
"sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
lp->id, lp->tx_ra, lp->tx_num_entries, hv_err);
hv_err = sun4v_ldc_tx_get_state(lp->id,
&lp->tx_head,
&lp->tx_tail,
&lp->chan_state);
if (hv_err)
printk(KERN_ERR PFX "ldc_abort: "
"sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n",
lp->id, hv_err);
hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
if (hv_err)
printk(KERN_ERR PFX "ldc_abort: "
"sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
lp->id, lp->rx_ra, lp->rx_num_entries, hv_err);
/* Refetch the RX queue state as well, because we could be invoked
* here in the queue processing context.
*/
hv_err = sun4v_ldc_rx_get_state(lp->id,
&lp->rx_head,
&lp->rx_tail,
&lp->chan_state);
if (hv_err)
printk(KERN_ERR PFX "ldc_abort: "
"sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n",
lp->id, hv_err);
return -ECONNRESET;
}
static struct ldc_version *find_by_major(u16 major)
{
struct ldc_version *ret = NULL;
int i;
for (i = 0; i < ARRAY_SIZE(ver_arr); i++) {
struct ldc_version *v = &ver_arr[i];
if (v->major <= major) {
ret = v;
break;
}
}
return ret;
}
static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp)
{
struct ldc_version *vap;
int err;
ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n",
vp->major, vp->minor);
if (lp->hs_state == LDC_HS_GOTVERS) {
lp->hs_state = LDC_HS_OPEN;
memset(&lp->ver, 0, sizeof(lp->ver));
}
vap = find_by_major(vp->major);
if (!vap) {
err = send_version_nack(lp, 0, 0);
} else if (vap->major != vp->major) {
err = send_version_nack(lp, vap->major, vap->minor);
} else {
struct ldc_version ver = *vp;
if (ver.minor > vap->minor)
ver.minor = vap->minor;
err = send_version_ack(lp, &ver);
if (!err) {
lp->ver = ver;
lp->hs_state = LDC_HS_GOTVERS;
}
}
if (err)
return LDC_ABORT(lp);
return 0;
}
static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp)
{
ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n",
vp->major, vp->minor);
if (lp->hs_state == LDC_HS_GOTVERS) {
if (lp->ver.major != vp->major ||
lp->ver.minor != vp->minor)
return LDC_ABORT(lp);
} else {
lp->ver = *vp;
lp->hs_state = LDC_HS_GOTVERS;
}
if (send_rts(lp))
return LDC_ABORT(lp);
return 0;
}
static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp)
{
struct ldc_version *vap;
struct ldc_packet *p;
unsigned long new_tail;
if (vp->major == 0 && vp->minor == 0)
return LDC_ABORT(lp);
vap = find_by_major(vp->major);
if (!vap)
return LDC_ABORT(lp);
p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
vap, sizeof(*vap),
&new_tail);
if (!p)
return LDC_ABORT(lp);
return send_tx_packet(lp, p, new_tail);
}
static int process_version(struct ldc_channel *lp,
struct ldc_packet *p)
{
struct ldc_version *vp;
vp = (struct ldc_version *) p->u.u_data;
switch (p->stype) {
case LDC_INFO:
return process_ver_info(lp, vp);
case LDC_ACK:
return process_ver_ack(lp, vp);
case LDC_NACK:
return process_ver_nack(lp, vp);
default:
return LDC_ABORT(lp);
}
}
static int process_rts(struct ldc_channel *lp,
struct ldc_packet *p)
{
ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n",
p->stype, p->seqid, p->env);
if (p->stype != LDC_INFO ||
lp->hs_state != LDC_HS_GOTVERS ||
p->env != lp->cfg.mode)
return LDC_ABORT(lp);
lp->snd_nxt = p->seqid;
lp->rcv_nxt = p->seqid;
lp->hs_state = LDC_HS_SENTRTR;
if (send_rtr(lp))
return LDC_ABORT(lp);
return 0;
}
static int process_rtr(struct ldc_channel *lp,
struct ldc_packet *p)
{
ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n",
p->stype, p->seqid, p->env);
if (p->stype != LDC_INFO ||
p->env != lp->cfg.mode)
return LDC_ABORT(lp);
lp->snd_nxt = p->seqid;
lp->hs_state = LDC_HS_COMPLETE;
ldc_set_state(lp, LDC_STATE_CONNECTED);
send_rdx(lp);
return LDC_EVENT_UP;
}
static int rx_seq_ok(struct ldc_channel *lp, u32 seqid)
{
return lp->rcv_nxt + 1 == seqid;
}
static int process_rdx(struct ldc_channel *lp,
struct ldc_packet *p)
{
ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n",
p->stype, p->seqid, p->env, p->u.r.ackid);
if (p->stype != LDC_INFO ||
!(rx_seq_ok(lp, p->seqid)))
return LDC_ABORT(lp);
lp->rcv_nxt = p->seqid;
lp->hs_state = LDC_HS_COMPLETE;
ldc_set_state(lp, LDC_STATE_CONNECTED);
return LDC_EVENT_UP;
}
static int process_control_frame(struct ldc_channel *lp,
struct ldc_packet *p)
{
switch (p->ctrl) {
case LDC_VERS:
return process_version(lp, p);
case LDC_RTS:
return process_rts(lp, p);
case LDC_RTR:
return process_rtr(lp, p);
case LDC_RDX:
return process_rdx(lp, p);
default:
return LDC_ABORT(lp);
}
}
static int process_error_frame(struct ldc_channel *lp,
struct ldc_packet *p)
{
return LDC_ABORT(lp);
}
static int process_data_ack(struct ldc_channel *lp,
struct ldc_packet *ack)
{
unsigned long head = lp->tx_acked;
u32 ackid = ack->u.r.ackid;
while (1) {
struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE);
head = tx_advance(lp, head);
if (p->seqid == ackid) {
lp->tx_acked = head;
return 0;
}
if (head == lp->tx_tail)
return LDC_ABORT(lp);
}
return 0;
}
static void send_events(struct ldc_channel *lp, unsigned int event_mask)
{
if (event_mask & LDC_EVENT_RESET)
lp->cfg.event(lp->event_arg, LDC_EVENT_RESET);
if (event_mask & LDC_EVENT_UP)
lp->cfg.event(lp->event_arg, LDC_EVENT_UP);
if (event_mask & LDC_EVENT_DATA_READY)
lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY);
}
static irqreturn_t ldc_rx(int irq, void *dev_id)
{
struct ldc_channel *lp = dev_id;
unsigned long orig_state, flags;
unsigned int event_mask;
spin_lock_irqsave(&lp->lock, flags);
orig_state = lp->chan_state;
/* We should probably check for hypervisor errors here and
* reset the LDC channel if we get one.
*/
sun4v_ldc_rx_get_state(lp->id,
&lp->rx_head,
&lp->rx_tail,
&lp->chan_state);
ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
orig_state, lp->chan_state, lp->rx_head, lp->rx_tail);
event_mask = 0;
if (lp->cfg.mode == LDC_MODE_RAW &&
lp->chan_state == LDC_CHANNEL_UP) {
lp->hs_state = LDC_HS_COMPLETE;
ldc_set_state(lp, LDC_STATE_CONNECTED);
/*
* Generate an LDC_EVENT_UP event if the channel
* was not already up.
*/
if (orig_state != LDC_CHANNEL_UP) {
event_mask |= LDC_EVENT_UP;
orig_state = lp->chan_state;
}
}
/* If we are in reset state, flush the RX queue and ignore
* everything.
*/
if (lp->flags & LDC_FLAG_RESET) {
(void) ldc_rx_reset(lp);
goto out;
}
/* Once we finish the handshake, we let the ldc_read()
* paths do all of the control frame and state management.
* Just trigger the callback.
*/
if (lp->hs_state == LDC_HS_COMPLETE) {
handshake_complete:
if (lp->chan_state != orig_state) {
unsigned int event = LDC_EVENT_RESET;
if (lp->chan_state == LDC_CHANNEL_UP)
event = LDC_EVENT_UP;
event_mask |= event;
}
if (lp->rx_head != lp->rx_tail)
event_mask |= LDC_EVENT_DATA_READY;
goto out;
}
if (lp->chan_state != orig_state)
goto out;
while (lp->rx_head != lp->rx_tail) {
struct ldc_packet *p;
unsigned long new;
int err;
p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
switch (p->type) {
case LDC_CTRL:
err = process_control_frame(lp, p);
if (err > 0)
event_mask |= err;
break;
case LDC_DATA:
event_mask |= LDC_EVENT_DATA_READY;
err = 0;
break;
case LDC_ERR:
err = process_error_frame(lp, p);
break;
default:
err = LDC_ABORT(lp);
break;
}
if (err < 0)
break;
new = lp->rx_head;
new += LDC_PACKET_SIZE;
if (new == (lp->rx_num_entries * LDC_PACKET_SIZE))
new = 0;
lp->rx_head = new;
err = __set_rx_head(lp, new);
if (err < 0) {
(void) LDC_ABORT(lp);
break;
}
if (lp->hs_state == LDC_HS_COMPLETE)
goto handshake_complete;
}
out:
spin_unlock_irqrestore(&lp->lock, flags);
send_events(lp, event_mask);
return IRQ_HANDLED;
}
static irqreturn_t ldc_tx(int irq, void *dev_id)
{
struct ldc_channel *lp = dev_id;
unsigned long flags, orig_state;
unsigned int event_mask = 0;
spin_lock_irqsave(&lp->lock, flags);
orig_state = lp->chan_state;
/* We should probably check for hypervisor errors here and
* reset the LDC channel if we get one.
*/
sun4v_ldc_tx_get_state(lp->id,
&lp->tx_head,
&lp->tx_tail,
&lp->chan_state);
ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
orig_state, lp->chan_state, lp->tx_head, lp->tx_tail);
if (lp->cfg.mode == LDC_MODE_RAW &&
lp->chan_state == LDC_CHANNEL_UP) {
lp->hs_state = LDC_HS_COMPLETE;
ldc_set_state(lp, LDC_STATE_CONNECTED);
/*
* Generate an LDC_EVENT_UP event if the channel
* was not already up.
*/
if (orig_state != LDC_CHANNEL_UP) {
event_mask |= LDC_EVENT_UP;
orig_state = lp->chan_state;
}
}
spin_unlock_irqrestore(&lp->lock, flags);
send_events(lp, event_mask);
return IRQ_HANDLED;
}
/* XXX ldc_alloc() and ldc_free() needs to run under a mutex so
* XXX that addition and removal from the ldc_channel_list has
* XXX atomicity, otherwise the __ldc_channel_exists() check is
* XXX totally pointless as another thread can slip into ldc_alloc()
* XXX and add a channel with the same ID. There also needs to be
* XXX a spinlock for ldc_channel_list.
*/
static HLIST_HEAD(ldc_channel_list);
static int __ldc_channel_exists(unsigned long id)
{
struct ldc_channel *lp;
hlist_for_each_entry(lp, &ldc_channel_list, list) {
if (lp->id == id)
return 1;
}
return 0;
}
static int alloc_queue(const char *name, unsigned long num_entries,
struct ldc_packet **base, unsigned long *ra)
{
unsigned long size, order;
void *q;
size = num_entries * LDC_PACKET_SIZE;
order = get_order(size);
q = (void *) __get_free_pages(GFP_KERNEL, order);
if (!q) {
printk(KERN_ERR PFX "Alloc of %s queue failed with "
"size=%lu order=%lu\n", name, size, order);
return -ENOMEM;
}
memset(q, 0, PAGE_SIZE << order);
*base = q;
*ra = __pa(q);
return 0;
}
static void free_queue(unsigned long num_entries, struct ldc_packet *q)
{
unsigned long size, order;
if (!q)
return;
size = num_entries * LDC_PACKET_SIZE;
order = get_order(size);
free_pages((unsigned long)q, order);
}
static unsigned long ldc_cookie_to_index(u64 cookie, void *arg)
{
u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;
/* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */
cookie &= ~COOKIE_PGSZ_CODE;
return (cookie >> (13ULL + (szcode * 3ULL)));
}
static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie,
unsigned long entry, unsigned long npages)
{
struct ldc_mtable_entry *base;
unsigned long i, shift;
shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3;
base = iommu->page_table + entry;
for (i = 0; i < npages; i++) {
if (base->cookie)
sun4v_ldc_revoke(id, cookie + (i << shift),
base->cookie);
base->mte = 0;
}
}
/* XXX Make this configurable... XXX */
#define LDC_IOTABLE_SIZE (8 * 1024)
static int ldc_iommu_init(const char *name, struct ldc_channel *lp)
{
unsigned long sz, num_tsb_entries, tsbsize, order;
struct ldc_iommu *ldc_iommu = &lp->iommu;
struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
struct ldc_mtable_entry *table;
unsigned long hv_err;
int err;
num_tsb_entries = LDC_IOTABLE_SIZE;
tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
spin_lock_init(&ldc_iommu->lock);
sz = num_tsb_entries / 8;
sz = (sz + 7UL) & ~7UL;
iommu->map = kzalloc(sz, GFP_KERNEL);
if (!iommu->map) {
printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz);
return -ENOMEM;
}
iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT,
NULL, false /* no large pool */,
1 /* npools */,
true /* skip span boundary check */);
order = get_order(tsbsize);
table = (struct ldc_mtable_entry *)
__get_free_pages(GFP_KERNEL, order);
err = -ENOMEM;
if (!table) {
printk(KERN_ERR PFX "Alloc of MTE table failed, "
"size=%lu order=%lu\n", tsbsize, order);
goto out_free_map;
}
memset(table, 0, PAGE_SIZE << order);
ldc_iommu->page_table = table;
hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table),
num_tsb_entries);
err = -EINVAL;
if (hv_err)
goto out_free_table;
return 0;
out_free_table:
free_pages((unsigned long) table, order);
ldc_iommu->page_table = NULL;
out_free_map:
kfree(iommu->map);
iommu->map = NULL;
return err;
}
static void ldc_iommu_release(struct ldc_channel *lp)
{
struct ldc_iommu *ldc_iommu = &lp->iommu;
struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
unsigned long num_tsb_entries, tsbsize, order;
(void) sun4v_ldc_set_map_table(lp->id, 0, 0);
num_tsb_entries = iommu->poolsize * iommu->nr_pools;
tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
order = get_order(tsbsize);
free_pages((unsigned long) ldc_iommu->page_table, order);
ldc_iommu->page_table = NULL;
kfree(iommu->map);
iommu->map = NULL;
}
struct ldc_channel *ldc_alloc(unsigned long id,
const struct ldc_channel_config *cfgp,
void *event_arg,
const char *name)
{
struct ldc_channel *lp;
const struct ldc_mode_ops *mops;
unsigned long dummy1, dummy2, hv_err;
u8 mss, *mssbuf;
int err;
err = -ENODEV;
if (!ldom_domaining_enabled)
goto out_err;
err = -EINVAL;
if (!cfgp)
goto out_err;
if (!name)
goto out_err;
switch (cfgp->mode) {
case LDC_MODE_RAW:
mops = &raw_ops;
mss = LDC_PACKET_SIZE;
break;
case LDC_MODE_UNRELIABLE:
mops = &nonraw_ops;
mss = LDC_PACKET_SIZE - 8;
break;
case LDC_MODE_STREAM:
mops = &stream_ops;
mss = LDC_PACKET_SIZE - 8 - 8;
break;
default:
goto out_err;
}
if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq)
goto out_err;
hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2);
err = -ENODEV;
if (hv_err == HV_ECHANNEL)
goto out_err;
err = -EEXIST;
if (__ldc_channel_exists(id))
goto out_err;
mssbuf = NULL;
lp = kzalloc(sizeof(*lp), GFP_KERNEL);
err = -ENOMEM;
if (!lp)
goto out_err;
spin_lock_init(&lp->lock);
lp->id = id;
err = ldc_iommu_init(name, lp);
if (err)
goto out_free_ldc;
lp->mops = mops;
lp->mss = mss;
lp->cfg = *cfgp;
if (!lp->cfg.mtu)
lp->cfg.mtu = LDC_DEFAULT_MTU;
if (lp->cfg.mode == LDC_MODE_STREAM) {
mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL);
if (!mssbuf) {
err = -ENOMEM;
goto out_free_iommu;
}
lp->mssbuf = mssbuf;
}
lp->event_arg = event_arg;
/* XXX allow setting via ldc_channel_config to override defaults
* XXX or use some formula based upon mtu
*/
lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
err = alloc_queue("TX", lp->tx_num_entries,
&lp->tx_base, &lp->tx_ra);
if (err)
goto out_free_mssbuf;
err = alloc_queue("RX", lp->rx_num_entries,
&lp->rx_base, &lp->rx_ra);
if (err)
goto out_free_txq;
lp->flags |= LDC_FLAG_ALLOCED_QUEUES;
lp->hs_state = LDC_HS_CLOSED;
ldc_set_state(lp, LDC_STATE_INIT);
INIT_HLIST_NODE(&lp->list);
hlist_add_head(&lp->list, &ldc_channel_list);
INIT_HLIST_HEAD(&lp->mh_list);
snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name);
snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name);
err = request_irq(lp->cfg.rx_irq, ldc_rx, 0,
lp->rx_irq_name, lp);
if (err)
goto out_free_txq;
err = request_irq(lp->cfg.tx_irq, ldc_tx, 0,
lp->tx_irq_name, lp);
if (err) {
free_irq(lp->cfg.rx_irq, lp);
goto out_free_txq;
}
return lp;
out_free_txq:
free_queue(lp->tx_num_entries, lp->tx_base);
out_free_mssbuf:
kfree(mssbuf);
out_free_iommu:
ldc_iommu_release(lp);
out_free_ldc:
kfree(lp);
out_err:
return ERR_PTR(err);
}
EXPORT_SYMBOL(ldc_alloc);
void ldc_unbind(struct ldc_channel *lp)
{
if (lp->flags & LDC_FLAG_REGISTERED_IRQS) {
free_irq(lp->cfg.rx_irq, lp);
free_irq(lp->cfg.tx_irq, lp);
lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
}
if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) {
sun4v_ldc_tx_qconf(lp->id, 0, 0);
sun4v_ldc_rx_qconf(lp->id, 0, 0);
lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
}
if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) {
free_queue(lp->tx_num_entries, lp->tx_base);
free_queue(lp->rx_num_entries, lp->rx_base);
lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES;
}
ldc_set_state(lp, LDC_STATE_INIT);
}
EXPORT_SYMBOL(ldc_unbind);
void ldc_free(struct ldc_channel *lp)
{
ldc_unbind(lp);
hlist_del(&lp->list);
kfree(lp->mssbuf);
ldc_iommu_release(lp);
kfree(lp);
}
EXPORT_SYMBOL(ldc_free);
/* Bind the channel. This registers the LDC queues with
* the hypervisor and puts the channel into a pseudo-listening
* state. This does not initiate a handshake, ldc_connect() does
* that.
*/
int ldc_bind(struct ldc_channel *lp)
{
unsigned long hv_err, flags;
int err = -EINVAL;
if (lp->state != LDC_STATE_INIT)
return -EINVAL;
spin_lock_irqsave(&lp->lock, flags);
enable_irq(lp->cfg.rx_irq);
enable_irq(lp->cfg.tx_irq);
lp->flags |= LDC_FLAG_REGISTERED_IRQS;
err = -ENODEV;
hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
if (hv_err)
goto out_free_irqs;
hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
if (hv_err)
goto out_free_irqs;
hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
if (hv_err)
goto out_unmap_tx;
hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
if (hv_err)
goto out_unmap_tx;
lp->flags |= LDC_FLAG_REGISTERED_QUEUES;
hv_err = sun4v_ldc_tx_get_state(lp->id,
&lp->tx_head,
&lp->tx_tail,
&lp->chan_state);
err = -EBUSY;
if (hv_err)
goto out_unmap_rx;
lp->tx_acked = lp->tx_head;
lp->hs_state = LDC_HS_OPEN;
ldc_set_state(lp, LDC_STATE_BOUND);
if (lp->cfg.mode == LDC_MODE_RAW) {
/*
* There is no handshake in RAW mode, so handshake
* is completed.
*/
lp->hs_state = LDC_HS_COMPLETE;
}
spin_unlock_irqrestore(&lp->lock, flags);
return 0;
out_unmap_rx:
lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
sun4v_ldc_rx_qconf(lp->id, 0, 0);
out_unmap_tx:
sun4v_ldc_tx_qconf(lp->id, 0, 0);
out_free_irqs:
lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
free_irq(lp->cfg.tx_irq, lp);
free_irq(lp->cfg.rx_irq, lp);
spin_unlock_irqrestore(&lp->lock, flags);
return err;
}
EXPORT_SYMBOL(ldc_bind);
int ldc_connect(struct ldc_channel *lp)
{
unsigned long flags;
int err;
if (lp->cfg.mode == LDC_MODE_RAW)
return -EINVAL;
spin_lock_irqsave(&lp->lock, flags);
if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
!(lp->flags & LDC_FLAG_REGISTERED_QUEUES) ||
lp->hs_state != LDC_HS_OPEN)
err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL);
else
err = start_handshake(lp);
spin_unlock_irqrestore(&lp->lock, flags);
return err;
}
EXPORT_SYMBOL(ldc_connect);
int ldc_disconnect(struct ldc_channel *lp)
{
unsigned long hv_err, flags;
int err;
if (lp->cfg.mode == LDC_MODE_RAW)
return -EINVAL;
if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
!(lp->flags & LDC_FLAG_REGISTERED_QUEUES))
return -EINVAL;
spin_lock_irqsave(&lp->lock, flags);
err = -ENODEV;
hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
if (hv_err)
goto out_err;
hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
if (hv_err)
goto out_err;
hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
if (hv_err)
goto out_err;
hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
if (hv_err)
goto out_err;
ldc_set_state(lp, LDC_STATE_BOUND);
lp->hs_state = LDC_HS_OPEN;
lp->flags |= LDC_FLAG_RESET;
spin_unlock_irqrestore(&lp->lock, flags);
return 0;
out_err:
sun4v_ldc_tx_qconf(lp->id, 0, 0);
sun4v_ldc_rx_qconf(lp->id, 0, 0);
free_irq(lp->cfg.tx_irq, lp);
free_irq(lp->cfg.rx_irq, lp);
lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS |
LDC_FLAG_REGISTERED_QUEUES);
ldc_set_state(lp, LDC_STATE_INIT);
spin_unlock_irqrestore(&lp->lock, flags);
return err;
}
EXPORT_SYMBOL(ldc_disconnect);
int ldc_state(struct ldc_channel *lp)
{
return lp->state;
}
EXPORT_SYMBOL(ldc_state);
void ldc_set_state(struct ldc_channel *lp, u8 state)
{
ldcdbg(STATE, "STATE (%s) --> (%s)\n",
state_to_str(lp->state),
state_to_str(state));
lp->state = state;
}
EXPORT_SYMBOL(ldc_set_state);
int ldc_mode(struct ldc_channel *lp)
{
return lp->cfg.mode;
}
EXPORT_SYMBOL(ldc_mode);
int ldc_rx_reset(struct ldc_channel *lp)
{
return __set_rx_head(lp, lp->rx_tail);
}
EXPORT_SYMBOL(ldc_rx_reset);
void __ldc_print(struct ldc_channel *lp, const char *caller)
{
pr_info("%s: id=0x%lx flags=0x%x state=%s cstate=0x%lx hsstate=0x%x\n"
"\trx_h=0x%lx rx_t=0x%lx rx_n=%ld\n"
"\ttx_h=0x%lx tx_t=0x%lx tx_n=%ld\n"
"\trcv_nxt=%u snd_nxt=%u\n",
caller, lp->id, lp->flags, state_to_str(lp->state),
lp->chan_state, lp->hs_state,
lp->rx_head, lp->rx_tail, lp->rx_num_entries,
lp->tx_head, lp->tx_tail, lp->tx_num_entries,
lp->rcv_nxt, lp->snd_nxt);
}
EXPORT_SYMBOL(__ldc_print);
static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size)
{
struct ldc_packet *p;
unsigned long new_tail, hv_err;
int err;
hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
&lp->chan_state);
if (unlikely(hv_err))
return -EBUSY;
if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
return LDC_ABORT(lp);
if (size > LDC_PACKET_SIZE)
return -EMSGSIZE;
p = data_get_tx_packet(lp, &new_tail);
if (!p)
return -EAGAIN;
memcpy(p, buf, size);
err = send_tx_packet(lp, p, new_tail);
if (!err)
err = size;
return err;
}
static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size)
{
struct ldc_packet *p;
unsigned long hv_err, new;
int err;
if (size < LDC_PACKET_SIZE)
return -EINVAL;
hv_err = sun4v_ldc_rx_get_state(lp->id,
&lp->rx_head,
&lp->rx_tail,
&lp->chan_state);
if (hv_err)
return LDC_ABORT(lp);
if (lp->chan_state == LDC_CHANNEL_DOWN ||
lp->chan_state == LDC_CHANNEL_RESETTING)
return -ECONNRESET;
if (lp->rx_head == lp->rx_tail)
return 0;
p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
memcpy(buf, p, LDC_PACKET_SIZE);
new = rx_advance(lp, lp->rx_head);
lp->rx_head = new;
err = __set_rx_head(lp, new);
if (err < 0)
err = -ECONNRESET;
else
err = LDC_PACKET_SIZE;
return err;
}
static const struct ldc_mode_ops raw_ops = {
.write = write_raw,
.read = read_raw,
};
static int write_nonraw(struct ldc_channel *lp, const void *buf,
unsigned int size)
{
unsigned long hv_err, tail;
unsigned int copied;
u32 seq;
int err;
hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
&lp->chan_state);
if (unlikely(hv_err))
return -EBUSY;
if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
return LDC_ABORT(lp);
if (!tx_has_space_for(lp, size))
return -EAGAIN;
seq = lp->snd_nxt;
copied = 0;
tail = lp->tx_tail;
while (copied < size) {
struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE);
u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ?
p->u.u_data :
p->u.r.r_data);
int data_len;
p->type = LDC_DATA;
p->stype = LDC_INFO;
p->ctrl = 0;
data_len = size - copied;
if (data_len > lp->mss)
data_len = lp->mss;
BUG_ON(data_len > LDC_LEN);
p->env = (data_len |
(copied == 0 ? LDC_START : 0) |
(data_len == size - copied ? LDC_STOP : 0));
p->seqid = ++seq;
ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n",
p->type,
p->stype,
p->ctrl,
p->env,
p->seqid);
memcpy(data, buf, data_len);
buf += data_len;
copied += data_len;
tail = tx_advance(lp, tail);
}
err = set_tx_tail(lp, tail);
if (!err) {
lp->snd_nxt = seq;
err = size;
}
return err;
}
static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p,
struct ldc_packet *first_frag)
{
int err;
if (first_frag)
lp->rcv_nxt = first_frag->seqid - 1;
err = send_data_nack(lp, p);
if (err)
return err;
err = ldc_rx_reset(lp);
if (err < 0)
return LDC_ABORT(lp);
return 0;
}
static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p)
{
if (p->stype & LDC_ACK) {
int err = process_data_ack(lp, p);
if (err)
return err;
}
if (p->stype & LDC_NACK)
return LDC_ABORT(lp);
return 0;
}
static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head)
{
unsigned long dummy;
int limit = 1000;
ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n",
cur_head, lp->rx_head, lp->rx_tail);
while (limit-- > 0) {
unsigned long hv_err;
hv_err = sun4v_ldc_rx_get_state(lp->id,
&dummy,
&lp->rx_tail,
&lp->chan_state);
if (hv_err)
return LDC_ABORT(lp);
if (lp->chan_state == LDC_CHANNEL_DOWN ||
lp->chan_state == LDC_CHANNEL_RESETTING)
return -ECONNRESET;
if (cur_head != lp->rx_tail) {
ldcdbg(DATA, "DATA WAIT DONE "
"head[%lx] tail[%lx] chan_state[%lx]\n",
dummy, lp->rx_tail, lp->chan_state);
return 0;
}
udelay(1);
}
return -EAGAIN;
}
static int rx_set_head(struct ldc_channel *lp, unsigned long head)
{
int err = __set_rx_head(lp, head);
if (err < 0)
return LDC_ABORT(lp);
lp->rx_head = head;
return 0;
}
static void send_data_ack(struct ldc_channel *lp)
{
unsigned long new_tail;
struct ldc_packet *p;
p = data_get_tx_packet(lp, &new_tail);
if (likely(p)) {
int err;
memset(p, 0, sizeof(*p));
p->type = LDC_DATA;
p->stype = LDC_ACK;
p->ctrl = 0;
p->seqid = lp->snd_nxt + 1;
p->u.r.ackid = lp->rcv_nxt;
err = send_tx_packet(lp, p, new_tail);
if (!err)
lp->snd_nxt++;
}
}
static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size)
{
struct ldc_packet *first_frag;
unsigned long hv_err, new;
int err, copied;
hv_err = sun4v_ldc_rx_get_state(lp->id,
&lp->rx_head,
&lp->rx_tail,
&lp->chan_state);
if (hv_err)
return LDC_ABORT(lp);
if (lp->chan_state == LDC_CHANNEL_DOWN ||
lp->chan_state == LDC_CHANNEL_RESETTING)
return -ECONNRESET;
if (lp->rx_head == lp->rx_tail)
return 0;
first_frag = NULL;
copied = err = 0;
new = lp->rx_head;
while (1) {
struct ldc_packet *p;
int pkt_len;
BUG_ON(new == lp->rx_tail);
p = lp->rx_base + (new / LDC_PACKET_SIZE);
ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] "
"rcv_nxt[%08x]\n",
p->type,
p->stype,
p->ctrl,
p->env,
p->seqid,
p->u.r.ackid,
lp->rcv_nxt);
if (unlikely(!rx_seq_ok(lp, p->seqid))) {
err = rx_bad_seq(lp, p, first_frag);
copied = 0;
break;
}
if (p->type & LDC_CTRL) {
err = process_control_frame(lp, p);
if (err < 0)
break;
err = 0;
}
lp->rcv_nxt = p->seqid;
/*
* If this is a control-only packet, there is nothing
* else to do but advance the rx queue since the packet
* was already processed above.
*/
if (!(p->type & LDC_DATA)) {
new = rx_advance(lp, new);
break;
}
if (p->stype & (LDC_ACK | LDC_NACK)) {
err = data_ack_nack(lp, p);
if (err)
break;
}
if (!(p->stype & LDC_INFO)) {
new = rx_advance(lp, new);
err = rx_set_head(lp, new);
if (err)
break;
goto no_data;
}
pkt_len = p->env & LDC_LEN;
/* Every initial packet starts with the START bit set.
*
* Singleton packets will have both START+STOP set.
*
* Fragments will have START set in the first frame, STOP
* set in the last frame, and neither bit set in middle
* frames of the packet.
*
* Therefore if we are at the beginning of a packet and
* we don't see START, or we are in the middle of a fragmented
* packet and do see START, we are unsynchronized and should
* flush the RX queue.
*/
if ((first_frag == NULL && !(p->env & LDC_START)) ||
(first_frag != NULL && (p->env & LDC_START))) {
if (!first_frag)
new = rx_advance(lp, new);
err = rx_set_head(lp, new);
if (err)
break;
if (!first_frag)
goto no_data;
}
if (!first_frag)
first_frag = p;
if (pkt_len > size - copied) {
/* User didn't give us a big enough buffer,
* what to do? This is a pretty serious error.
*
* Since we haven't updated the RX ring head to
* consume any of the packets, signal the error
* to the user and just leave the RX ring alone.
*
* This seems the best behavior because this allows
* a user of the LDC layer to start with a small
* RX buffer for ldc_read() calls and use -EMSGSIZE
* as a cue to enlarge it's read buffer.
*/
err = -EMSGSIZE;
break;
}
/* Ok, we are gonna eat this one. */
new = rx_advance(lp, new);
memcpy(buf,
(lp->cfg.mode == LDC_MODE_UNRELIABLE ?
p->u.u_data : p->u.r.r_data), pkt_len);
buf += pkt_len;
copied += pkt_len;
if (p->env & LDC_STOP)
break;
no_data:
if (new == lp->rx_tail) {
err = rx_data_wait(lp, new);
if (err)
break;
}
}
if (!err)
err = rx_set_head(lp, new);
if (err && first_frag)
lp->rcv_nxt = first_frag->seqid - 1;
if (!err) {
err = copied;
if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE)
send_data_ack(lp);
}
return err;
}
static const struct ldc_mode_ops nonraw_ops = {
.write = write_nonraw,
.read = read_nonraw,
};
static int write_stream(struct ldc_channel *lp, const void *buf,
unsigned int size)
{
if (size > lp->cfg.mtu)
size = lp->cfg.mtu;
return write_nonraw(lp, buf, size);
}
static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size)
{
if (!lp->mssbuf_len) {
int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu);
if (err < 0)
return err;
lp->mssbuf_len = err;
lp->mssbuf_off = 0;
}
if (size > lp->mssbuf_len)
size = lp->mssbuf_len;
memcpy(buf, lp->mssbuf + lp->mssbuf_off, size);
lp->mssbuf_off += size;
lp->mssbuf_len -= size;
return size;
}
static const struct ldc_mode_ops stream_ops = {
.write = write_stream,
.read = read_stream,
};
int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size)
{
unsigned long flags;
int err;
if (!buf)
return -EINVAL;
if (!size)
return 0;
spin_lock_irqsave(&lp->lock, flags);
if (lp->hs_state != LDC_HS_COMPLETE)
err = -ENOTCONN;
else
err = lp->mops->write(lp, buf, size);
spin_unlock_irqrestore(&lp->lock, flags);
return err;
}
EXPORT_SYMBOL(ldc_write);
int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size)
{
unsigned long flags;
int err;
ldcdbg(RX, "%s: entered size=%d\n", __func__, size);
if (!buf)
return -EINVAL;
if (!size)
return 0;
spin_lock_irqsave(&lp->lock, flags);
if (lp->hs_state != LDC_HS_COMPLETE)
err = -ENOTCONN;
else
err = lp->mops->read(lp, buf, size);
spin_unlock_irqrestore(&lp->lock, flags);
ldcdbg(RX, "%s: mode=%d, head=%lu, tail=%lu rv=%d\n", __func__,
lp->cfg.mode, lp->rx_head, lp->rx_tail, err);
return err;
}
EXPORT_SYMBOL(ldc_read);
static u64 pagesize_code(void)
{
switch (PAGE_SIZE) {
default:
case (8ULL * 1024ULL):
return 0;
case (64ULL * 1024ULL):
return 1;
case (512ULL * 1024ULL):
return 2;
case (4ULL * 1024ULL * 1024ULL):
return 3;
case (32ULL * 1024ULL * 1024ULL):
return 4;
case (256ULL * 1024ULL * 1024ULL):
return 5;
}
}
static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset)
{
return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) |
(index << PAGE_SHIFT) |
page_offset);
}
static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,
unsigned long npages)
{
long entry;
entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table,
npages, NULL, (unsigned long)-1, 0);
if (unlikely(entry == IOMMU_ERROR_CODE))
return NULL;
return iommu->page_table + entry;
}
static u64 perm_to_mte(unsigned int map_perm)
{
u64 mte_base;
mte_base = pagesize_code();
if (map_perm & LDC_MAP_SHADOW) {
if (map_perm & LDC_MAP_R)
mte_base |= LDC_MTE_COPY_R;
if (map_perm & LDC_MAP_W)
mte_base |= LDC_MTE_COPY_W;
}
if (map_perm & LDC_MAP_DIRECT) {
if (map_perm & LDC_MAP_R)
mte_base |= LDC_MTE_READ;
if (map_perm & LDC_MAP_W)
mte_base |= LDC_MTE_WRITE;
if (map_perm & LDC_MAP_X)
mte_base |= LDC_MTE_EXEC;
}
if (map_perm & LDC_MAP_IO) {
if (map_perm & LDC_MAP_R)
mte_base |= LDC_MTE_IOMMU_R;
if (map_perm & LDC_MAP_W)
mte_base |= LDC_MTE_IOMMU_W;
}
return mte_base;
}
static int pages_in_region(unsigned long base, long len)
{
int count = 0;
do {
unsigned long new = (base + PAGE_SIZE) & PAGE_MASK;
len -= (new - base);
base = new;
count++;
} while (len > 0);
return count;
}
struct cookie_state {
struct ldc_mtable_entry *page_table;
struct ldc_trans_cookie *cookies;
u64 mte_base;
u64 prev_cookie;
u32 pte_idx;
u32 nc;
};
static void fill_cookies(struct cookie_state *sp, unsigned long pa,
unsigned long off, unsigned long len)
{
do {
unsigned long tlen, new = pa + PAGE_SIZE;
u64 this_cookie;
sp->page_table[sp->pte_idx].mte = sp->mte_base | pa;
tlen = PAGE_SIZE;
if (off)
tlen = PAGE_SIZE - off;
if (tlen > len)
tlen = len;
this_cookie = make_cookie(sp->pte_idx,
pagesize_code(), off);
off = 0;
if (this_cookie == sp->prev_cookie) {
sp->cookies[sp->nc - 1].cookie_size += tlen;
} else {
sp->cookies[sp->nc].cookie_addr = this_cookie;
sp->cookies[sp->nc].cookie_size = tlen;
sp->nc++;
}
sp->prev_cookie = this_cookie + tlen;
sp->pte_idx++;
len -= tlen;
pa = new;
} while (len > 0);
}
static int sg_count_one(struct scatterlist *sg)
{
unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT;
long len = sg->length;
if ((sg->offset | len) & (8UL - 1))
return -EFAULT;
return pages_in_region(base + sg->offset, len);
}
static int sg_count_pages(struct scatterlist *sg, int num_sg)
{
int count;
int i;
count = 0;
for (i = 0; i < num_sg; i++) {
int err = sg_count_one(sg + i);
if (err < 0)
return err;
count += err;
}
return count;
}
int ldc_map_sg(struct ldc_channel *lp,
struct scatterlist *sg, int num_sg,
struct ldc_trans_cookie *cookies, int ncookies,
unsigned int map_perm)
{
unsigned long i, npages;
struct ldc_mtable_entry *base;
struct cookie_state state;
struct ldc_iommu *iommu;
int err;
struct scatterlist *s;
if (map_perm & ~LDC_MAP_ALL)
return -EINVAL;
err = sg_count_pages(sg, num_sg);
if (err < 0)
return err;
npages = err;
if (err > ncookies)
return -EMSGSIZE;
iommu = &lp->iommu;
base = alloc_npages(iommu, npages);
if (!base)
return -ENOMEM;
state.page_table = iommu->page_table;
state.cookies = cookies;
state.mte_base = perm_to_mte(map_perm);
state.prev_cookie = ~(u64)0;
state.pte_idx = (base - iommu->page_table);
state.nc = 0;
for_each_sg(sg, s, num_sg, i) {
fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT,
s->offset, s->length);
}
return state.nc;
}
EXPORT_SYMBOL(ldc_map_sg);
int ldc_map_single(struct ldc_channel *lp,
void *buf, unsigned int len,
struct ldc_trans_cookie *cookies, int ncookies,
unsigned int map_perm)
{
unsigned long npages, pa;
struct ldc_mtable_entry *base;
struct cookie_state state;
struct ldc_iommu *iommu;
if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1))
return -EINVAL;
pa = __pa(buf);
if ((pa | len) & (8UL - 1))
return -EFAULT;
npages = pages_in_region(pa, len);
iommu = &lp->iommu;
base = alloc_npages(iommu, npages);
if (!base)
return -ENOMEM;
state.page_table = iommu->page_table;
state.cookies = cookies;
state.mte_base = perm_to_mte(map_perm);
state.prev_cookie = ~(u64)0;
state.pte_idx = (base - iommu->page_table);
state.nc = 0;
fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len);
BUG_ON(state.nc > ncookies);
return state.nc;
}
EXPORT_SYMBOL(ldc_map_single);
static void free_npages(unsigned long id, struct ldc_iommu *iommu,
u64 cookie, u64 size)
{
unsigned long npages, entry;
npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT;
entry = ldc_cookie_to_index(cookie, iommu);
ldc_demap(iommu, id, cookie, entry, npages);
iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry);
}
void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies,
int ncookies)
{
struct ldc_iommu *iommu = &lp->iommu;
int i;
unsigned long flags;
spin_lock_irqsave(&iommu->lock, flags);
for (i = 0; i < ncookies; i++) {
u64 addr = cookies[i].cookie_addr;
u64 size = cookies[i].cookie_size;
free_npages(lp->id, iommu, addr, size);
}
spin_unlock_irqrestore(&iommu->lock, flags);
}
EXPORT_SYMBOL(ldc_unmap);
int ldc_copy(struct ldc_channel *lp, int copy_dir,
void *buf, unsigned int len, unsigned long offset,
struct ldc_trans_cookie *cookies, int ncookies)
{
unsigned int orig_len;
unsigned long ra;
int i;
if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) {
printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n",
lp->id, copy_dir);
return -EINVAL;
}
ra = __pa(buf);
if ((ra | len | offset) & (8UL - 1)) {
printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer "
"ra[%lx] len[%x] offset[%lx]\n",
lp->id, ra, len, offset);
return -EFAULT;
}
if (lp->hs_state != LDC_HS_COMPLETE ||
(lp->flags & LDC_FLAG_RESET)) {
printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] "
"flags[%x]\n", lp->id, lp->hs_state, lp->flags);
return -ECONNRESET;
}
orig_len = len;
for (i = 0; i < ncookies; i++) {
unsigned long cookie_raddr = cookies[i].cookie_addr;
unsigned long this_len = cookies[i].cookie_size;
unsigned long actual_len;
if (unlikely(offset)) {
unsigned long this_off = offset;
if (this_off > this_len)
this_off = this_len;
offset -= this_off;
this_len -= this_off;
if (!this_len)
continue;
cookie_raddr += this_off;
}
if (this_len > len)
this_len = len;
while (1) {
unsigned long hv_err;
hv_err = sun4v_ldc_copy(lp->id, copy_dir,
cookie_raddr, ra,
this_len, &actual_len);
if (unlikely(hv_err)) {
printk(KERN_ERR PFX "ldc_copy: ID[%lu] "
"HV error %lu\n",
lp->id, hv_err);
if (lp->hs_state != LDC_HS_COMPLETE ||
(lp->flags & LDC_FLAG_RESET))
return -ECONNRESET;
else
return -EFAULT;
}
cookie_raddr += actual_len;
ra += actual_len;
len -= actual_len;
if (actual_len == this_len)
break;
this_len -= actual_len;
}
if (!len)
break;
}
/* It is caller policy what to do about short copies.
* For example, a networking driver can declare the
* packet a runt and drop it.
*/
return orig_len - len;
}
EXPORT_SYMBOL(ldc_copy);
void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len,
struct ldc_trans_cookie *cookies, int *ncookies,
unsigned int map_perm)
{
void *buf;
int err;
if (len & (8UL - 1))
return ERR_PTR(-EINVAL);
buf = kzalloc(len, GFP_ATOMIC);
if (!buf)
return ERR_PTR(-ENOMEM);
err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm);
if (err < 0) {
kfree(buf);
return ERR_PTR(err);
}
*ncookies = err;
return buf;
}
EXPORT_SYMBOL(ldc_alloc_exp_dring);
void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len,
struct ldc_trans_cookie *cookies, int ncookies)
{
ldc_unmap(lp, cookies, ncookies);
kfree(buf);
}
EXPORT_SYMBOL(ldc_free_exp_dring);
static int __init ldc_init(void)
{
unsigned long major, minor;
struct mdesc_handle *hp;
const u64 *v;
int err;
u64 mp;
hp = mdesc_grab();
if (!hp)
return -ENODEV;
mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform");
err = -ENODEV;
if (mp == MDESC_NODE_NULL)
goto out;
v = mdesc_get_property(hp, mp, "domaining-enabled", NULL);
if (!v)
goto out;
major = 1;
minor = 0;
if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) {
printk(KERN_INFO PFX "Could not register LDOM hvapi.\n");
goto out;
}
printk(KERN_INFO "%s", version);
if (!*v) {
printk(KERN_INFO PFX "Domaining disabled.\n");
goto out;
}
ldom_domaining_enabled = 1;
err = 0;
out:
mdesc_release(hp);
return err;
}
core_initcall(ldc_init);