#include <linux/jiffies.h>
#include <linux/math64.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/printk.h>
#include <linux/time.h>
#include <net/tcp.h>
#include <vdso/time64.h>
#define BICTCP_BETA_SCALE 1024
#define BICTCP_HZ 10 /* BIC HZ 2^10 = 1024 */
#define ROCCET_ALPHA_TIMES_100 20
#define ROCCET_NEXT_MIN_RTT_PROBE_MS 5000
#define ROCCET_MAX_PROBE_INTERVAL_MS 10000
#define ROCCET_MAX_SRRTT_CHECK_INTERVAL_MS (5 * 200)
enum roccet_state {
LAUNCH,
ORBITER,
RTT_PROBE_ENTER,
RTT_PROBE,
RTT_PROBE_REFILL,
DRAIN
};
struct roccettcp {
u32 cnt;
u32 last_max_cwnd;
u32 last_cwnd;
u32 last_time;
u32 bic_origin_point;
u32 bic_K;
u32 delay_min;
u32 epoch_start;
u32 ack_cnt;
u32 tcp_cwnd;
u32 curr_rtt;
u32 roccet_last_event_time_us;
u32 curr_min_rtt;
u32 next_min_rtt_probe;
u32 probe_min_rtt_until;
u32 refill_until;
u32 cwnd_before_min_rtt_probe;
u32 curr_srrtt;
u32 next_srrtt_check_ts;
u32 last_rtt;
u32 interval_snd_seq_start;
u32 interval_una_seq_start;
u32 ack_carry_over;
u32 last_ack_rate_time;
u16 ack_rate_last_rate;
u16 curr_ack_rate;
u16 ack_rate_cnt;
enum roccet_state state : 3;
bool initial_limit_reached: 1;
bool is_in_initial_launch: 1;
bool last_event_time_set: 1;
bool rtt_probe_timers_set: 1;
};
static uint sr_rtt_upper_bound __read_mostly = 100;
static int ack_rate_diff_ss __read_mostly = 10;
module_param(sr_rtt_upper_bound, uint, 0644);
MODULE_PARM_DESC(sr_rtt_upper_bound, "ROCCET's upper bound for srRTT.");
module_param(ack_rate_diff_ss, int, 0644);
MODULE_PARM_DESC(ack_rate_diff_ss,
"ROCCET's threshold to exit slow start if ACK-rate differs by given amount of segments.");
static int fast_convergence __read_mostly = 1;
static int beta __read_mostly = 717;
static int initial_ssthresh __read_mostly;
static int bic_scale __read_mostly = 41;
static int tcp_friendliness __read_mostly = 1;
static u32 cube_rtt_scale __read_mostly;
static u32 beta_scale __read_mostly;
static u64 cube_factor __read_mostly;
static int beta_param_set(const char *val, const struct kernel_param *kp);
static const struct kernel_param_ops beta_param_ops = {
.set = beta_param_set,
.get = param_get_int,
};
module_param_cb(beta, &beta_param_ops, &beta, 0644);
MODULE_PARM_DESC(beta, "beta factor for multiplicative decrease");
module_param(fast_convergence, int, 0644);
MODULE_PARM_DESC(fast_convergence, "turn on/off fast convergence");
module_param(initial_ssthresh, int, 0644);
MODULE_PARM_DESC(initial_ssthresh, "initial value of slow start threshold");
module_param(bic_scale, int, 0444);
MODULE_PARM_DESC(bic_scale,
"scale (scaled by 1024) value for bic function (bic_scale/1024)");
module_param(tcp_friendliness, int, 0644);
MODULE_PARM_DESC(tcp_friendliness, "turn on/off tcp friendliness");
static int beta_param_set(const char *val, const struct kernel_param *kp)
{
int beta_candidate;
int ret = kstrtoint(val, 10, &beta_candidate);
if (ret)
return ret;
if (beta_candidate <= 0 || beta_candidate >= BICTCP_BETA_SCALE) {
pr_err_once("TCP ROCCET: beta must be between 0 and %d\n",
BICTCP_BETA_SCALE);
return -EINVAL;
}
return param_set_int(val, kp);
}
static int param_check(void)
{
if (beta <= 0 || beta >= BICTCP_BETA_SCALE) {
pr_err_once("TCP ROCCET: beta must be between 0 and %d\n",
BICTCP_BETA_SCALE);
return -EINVAL;
}
if (bic_scale <= 0) {
pr_err_once("TCP ROCCET: bic_scale must be positive\n");
return -EINVAL;
}
return 0;
}
static void param_precompute(void)
{
beta_scale =
8 * (BICTCP_BETA_SCALE + beta) / 3 / (BICTCP_BETA_SCALE - beta);
cube_rtt_scale = (bic_scale * 10);
cube_factor = 1ull << (10 + 3 * BICTCP_HZ);
do_div(cube_factor, bic_scale * 10);
}
static void roccet_reset(struct sock *sk, struct roccettcp *ca)
{
bool prev_initial_limit_reached = ca->initial_limit_reached;
bool prev_is_in_initial_launch = ca->is_in_initial_launch;
memset(ca, 0, sizeof(struct roccettcp));
ca->curr_min_rtt = U32_MAX;
ca->curr_rtt = U32_MAX;
ca->last_rtt = U32_MAX;
ca->roccet_last_event_time_us = jiffies_to_usecs(tcp_jiffies32);
ca->last_ack_rate_time = jiffies_to_usecs(tcp_jiffies32);
ca->interval_snd_seq_start = tcp_sk(sk)->snd_nxt;
ca->interval_una_seq_start = tcp_sk(sk)->snd_una;
ca->state = LAUNCH;
ca->initial_limit_reached = prev_initial_limit_reached;
ca->is_in_initial_launch = prev_is_in_initial_launch;
}
static void roccet_init(struct sock *sk)
{
struct roccettcp *ca = inet_csk_ca(sk);
roccet_reset(sk, ca);
ca->initial_limit_reached = false;
ca->is_in_initial_launch = true;
if (initial_ssthresh)
WRITE_ONCE(tcp_sk(sk)->snd_ssthresh, initial_ssthresh);
cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED);
}
static void roccet_set_next_srrtt_check_ts(struct roccettcp *ca, u32 now)
{
ca->next_srrtt_check_ts = now + min(5 * ca->curr_rtt,
ROCCET_MAX_SRRTT_CHECK_INTERVAL_MS
* USEC_PER_MSEC);
}
static void roccet_set_next_min_rtt_probe_ts(struct roccettcp *ca)
{
if (ca->state == RTT_PROBE || ca->state == RTT_PROBE_REFILL)
return;
ca->next_min_rtt_probe = jiffies_to_usecs(tcp_jiffies32) +
ROCCET_NEXT_MIN_RTT_PROBE_MS * USEC_PER_MSEC;
}
static void update_min_rtt(struct roccettcp *ca)
{
if (ca->curr_rtt < ca->curr_min_rtt) {
ca->curr_min_rtt = max(ca->curr_rtt, 1);
roccet_set_next_min_rtt_probe_ts(ca);
}
}
static s32 get_ack_rate_diff(struct roccettcp *ca)
{
if (ca->curr_ack_rate < ca->ack_rate_last_rate)
return 0;
return (s32)(ca->curr_ack_rate - ca->ack_rate_last_rate);
}
static void update_ack_rate(struct roccettcp *ca, u32 acked, u32 now)
{
const s32 idle_threshold = USEC_PER_SEC * 2;
const s32 interval = USEC_PER_MSEC * 100;
bool interval_elapsed = !time_between32(now, ca->last_ack_rate_time
- interval,
ca->last_ack_rate_time
+ interval);
if (interval_elapsed) {
if (!time_between32(now, ca->last_ack_rate_time
- idle_threshold,
ca->last_ack_rate_time
+ idle_threshold)) {
ca->ack_rate_last_rate = 0;
ca->curr_ack_rate = 0;
} else {
ca->ack_rate_last_rate = ca->curr_ack_rate;
ca->curr_ack_rate = ca->ack_rate_cnt;
}
ca->ack_rate_cnt = min_t(u32, acked, U16_MAX);
ca->last_ack_rate_time = now;
} else {
ca->ack_rate_cnt = min_t(u32, ca->ack_rate_cnt + acked,
U16_MAX);
}
}
static void update_srrtt(struct roccettcp *ca)
{
u64 rrtt;
if (ca->curr_rtt < ca->curr_min_rtt) {
ca->curr_rtt = max(ca->curr_rtt, 1);
ca->curr_min_rtt = ca->curr_rtt;
}
if (ca->curr_min_rtt == 0) {
pr_err_once("TCP ROCCET: Recorded curr_min_rtt is 0\n");
return;
}
rrtt = div_u64(100 * (u64)(ca->curr_rtt - ca->curr_min_rtt),
ca->curr_min_rtt);
ca->curr_srrtt = div_u64((100 - ROCCET_ALPHA_TIMES_100)
* (u64)ca->curr_srrtt + ROCCET_ALPHA_TIMES_100
* rrtt, 100);
}
static void roccet_min_rtt_probe_ce(struct roccettcp *ca, u32 cwnd)
{
if (ca->state != RTT_PROBE)
return;
if (ca->cwnd_before_min_rtt_probe == 0)
pr_warn_once("TCP ROCCET: cwnd_before_min_rtt_probe is 0 during RTT_PROBE. This should not happen.\n");
else
cwnd = ca->cwnd_before_min_rtt_probe;
ca->cwnd_before_min_rtt_probe =
max((cwnd * beta) / BICTCP_BETA_SCALE, 2U);
}
static void roccet_congestion_event(struct sock *sk, u32 now)
{
struct tcp_sock *tp = tcp_sk(sk);
struct roccettcp *ca = inet_csk_ca(sk);
u32 curr_cwnd = tcp_snd_cwnd(tp);
ca->epoch_start = 0;
ca->roccet_last_event_time_us = now;
ca->last_event_time_set = true;
if (ca->state == RTT_PROBE) {
roccet_min_rtt_probe_ce(ca, curr_cwnd);
return;
}
ca->cnt = 100 * curr_cwnd;
if (ca->last_max_cwnd < curr_cwnd)
ca->last_max_cwnd = curr_cwnd;
tcp_snd_cwnd_set(tp, min(tp->snd_cwnd_clamp,
max((curr_cwnd * beta)
/ BICTCP_BETA_SCALE, 2U)));
WRITE_ONCE(tp->snd_ssthresh, tcp_snd_cwnd(tp));
if (ca->state == LAUNCH) {
ca->state = ORBITER;
} else if (ca->state == ORBITER || ca->state == RTT_PROBE_REFILL) {
ca->state = DRAIN;
}
}
static void roccet_enter_min_rtt_probe(struct sock *sk, u32 now)
{
struct tcp_sock *tp = tcp_sk(sk);
struct roccettcp *ca = inet_csk_ca(sk);
u32 interval, probe_cwnd;
if (ca->rtt_probe_timers_set)
pr_warn_once("TCP ROCCET: Probing time should not be set when entering RTT Probe\n");
if (ca->curr_rtt == U32_MAX) {
pr_warn_once("TCP ROCCET: curr_rtt is U32_MAX. Won't enter RTT_PROBE\n");
ca->state = ORBITER;
return;
}
ca->next_min_rtt_probe = tcp_jiffies32;
interval = max(200 * USEC_PER_MSEC, min(ca->curr_rtt,
ROCCET_MAX_PROBE_INTERVAL_MS
* USEC_PER_MSEC));
if (!tcp_is_cwnd_limited(sk))
probe_cwnd = max(tcp_snd_cwnd(tp) / 3, TCP_INIT_CWND);
else
probe_cwnd = max(tcp_snd_cwnd(tp) / 2, TCP_INIT_CWND);
ca->probe_min_rtt_until = now + interval;
ca->cwnd_before_min_rtt_probe = tcp_snd_cwnd(tp);
tcp_snd_cwnd_set(tp, min(probe_cwnd, tp->snd_cwnd_clamp));
ca->curr_min_rtt = U32_MAX;
ca->refill_until = ca->probe_min_rtt_until + interval;
ca->rtt_probe_timers_set = true;
ca->state = RTT_PROBE;
}
static void roccet_min_rtt_probe(struct sock *sk, u32 now)
{
struct tcp_sock *tp = tcp_sk(sk);
struct roccettcp *ca = inet_csk_ca(sk);
if (!ca->rtt_probe_timers_set)
pr_warn_once("TCP ROCCET: Probing time should be set\n");
else if (time_between32(now, ca->probe_min_rtt_until
- ROCCET_MAX_PROBE_INTERVAL_MS
* USEC_PER_MSEC, ca->probe_min_rtt_until))
return;
else if (time_after32(now, ca->refill_until))
pr_warn_once("TCP ROCCET: Skipped refill interval.\n");
if (ca->cwnd_before_min_rtt_probe == 0)
pr_warn_once("TCP ROCCET: cwnd_before_min_rtt_probe is 0 during RTT_PROBE.\n");
else
tcp_snd_cwnd_set(tp, min(ca->cwnd_before_min_rtt_probe,
tp->snd_cwnd_clamp));
ca->cwnd_before_min_rtt_probe = 0;
ca->state = RTT_PROBE_REFILL;
}
static void roccet_rtt_probe_refill(struct roccettcp *ca, u32 now)
{
if (!time_between32(now, ca->refill_until
- ROCCET_MAX_PROBE_INTERVAL_MS
* USEC_PER_MSEC, ca->refill_until)) {
ca->state = ORBITER;
}
}
static u32 cubic_root(u64 a)
{
u32 x, b, shift;
static const u8 v[] = {
0, 54, 54, 54, 118, 118, 118, 118,
123, 129, 134, 138, 143, 147, 151, 156,
157, 161, 164, 168, 170, 173, 176, 179,
181, 185, 187, 190, 192, 194, 197, 199,
200, 202, 204, 206, 209, 211, 213, 215,
217, 219, 221, 222, 224, 225, 227, 229,
231, 232, 234, 236, 237, 239, 240, 242,
244, 245, 246, 248, 250, 251, 252, 254,
};
b = fls64(a);
if (b < 7) {
return ((u32)v[(u32)a] + 35) >> 6;
}
b = ((b * 84) >> 8) - 1;
shift = (a >> (b * 3));
x = ((u32)(((u32)v[shift] + 10) << b)) >> 6;
x = (2 * x + (u32)div64_u64(a, (u64)x * (u64)(x - 1)));
x = ((x * 341) >> 10);
return x;
}
static void bictcp_update(struct roccettcp *ca, u32 cwnd,
u32 acked)
{
u32 delta, bic_target, max_cnt;
u64 offs, t;
ca->ack_cnt += acked;
if (ca->last_cwnd == cwnd &&
(s32)(tcp_jiffies32 - ca->last_time) <= HZ / 32)
return;
if (ca->epoch_start && tcp_jiffies32 == ca->last_time)
goto tcp_friendliness;
ca->last_cwnd = cwnd;
ca->last_time = tcp_jiffies32;
if (ca->epoch_start == 0) {
ca->epoch_start = tcp_jiffies32;
ca->ack_cnt = acked;
ca->tcp_cwnd = cwnd;
if (ca->last_max_cwnd <= cwnd) {
ca->bic_K = 0;
ca->bic_origin_point = cwnd;
} else {
ca->bic_K = cubic_root(cube_factor *
(ca->last_max_cwnd - cwnd));
ca->bic_origin_point = ca->last_max_cwnd;
}
}
t = (s32)(tcp_jiffies32 - ca->epoch_start);
t += usecs_to_jiffies(ca->delay_min);
t <<= BICTCP_HZ;
do_div(t, HZ);
if (t < ca->bic_K)
offs = ca->bic_K - t;
else
offs = t - ca->bic_K;
delta = (cube_rtt_scale * offs * offs * offs) >> (10 + 3 * BICTCP_HZ);
if (t < ca->bic_K)
bic_target = ca->bic_origin_point - delta;
else
bic_target = ca->bic_origin_point + delta;
if (bic_target > cwnd)
ca->cnt = cwnd / (bic_target - cwnd);
else
ca->cnt = 100 * cwnd;
if (ca->last_max_cwnd == 0 && ca->cnt > 20)
ca->cnt = 20;
tcp_friendliness:
if (tcp_friendliness) {
u32 scale = beta_scale;
delta = (cwnd * scale) >> 3;
if (delta > 0) {
while (ca->ack_cnt > delta) {
ca->ack_cnt -= delta;
ca->tcp_cwnd++;
}
}
if (ca->tcp_cwnd > cwnd) {
delta = ca->tcp_cwnd - cwnd;
max_cnt = cwnd / delta;
if (ca->cnt > max_cnt)
ca->cnt = max_cnt;
}
}
ca->cnt = max(ca->cnt, 2U);
}
static void roccet_launch_update(struct sock *sk, u32 acked)
{
struct tcp_sock *tp = tcp_sk(sk);
struct roccettcp *ca = inet_csk_ca(sk);
u32 now = jiffies_to_usecs(tcp_jiffies32);
if ((ca->curr_srrtt > sr_rtt_upper_bound &&
get_ack_rate_diff(ca) <= ack_rate_diff_ss) ||
!tcp_in_slow_start(tp) ||
(!tcp_is_cwnd_limited(sk) && ca->initial_limit_reached)) {
u32 new_cwnd;
ca->epoch_start = 0;
if (ca->is_in_initial_launch) {
new_cwnd = tcp_snd_cwnd(tp) / 2;
} else {
new_cwnd = tcp_snd_cwnd(tp) - (tcp_snd_cwnd(tp) / 3);
}
tcp_snd_cwnd_set(tp, min(max(new_cwnd, TCP_INIT_CWND),
tp->snd_cwnd_clamp));
WRITE_ONCE(tp->snd_ssthresh, tcp_snd_cwnd(tp));
ca->roccet_last_event_time_us = now;
ca->last_event_time_set = true;
ca->state = ORBITER;
return;
}
acked = tcp_slow_start(tp, acked);
if (!tcp_in_slow_start(tp)) {
ca->state = ORBITER;
ca->epoch_start = 0;
ca->ack_carry_over = acked;
}
}
static void roccet_orbiter_update(struct sock *sk, u32 acked)
{
struct tcp_sock *tp = tcp_sk(sk);
struct roccettcp *ca = inet_csk_ca(sk);
u32 now = jiffies_to_usecs(tcp_jiffies32);
bool evaluate_srrtt = false;
bool sent_more_than_acked = false;
u32 roccet_xj, jitter, sent, received;
if (ca->last_event_time_set &&
time_between32(now, ca->roccet_last_event_time_us
- 100 * USEC_PER_MSEC,
ca->roccet_last_event_time_us
+ 100 * USEC_PER_MSEC)) {
ca->state = DRAIN;
return;
}
if (!time_between32(now, ca->next_min_rtt_probe
- 2 * ROCCET_NEXT_MIN_RTT_PROBE_MS
* USEC_PER_MSEC, ca->next_min_rtt_probe) &&
ca->curr_rtt != U32_MAX) {
ca->state = RTT_PROBE_ENTER;
return;
}
if ((s32)(ca->curr_rtt - ca->last_rtt) < 0)
jitter = ca->last_rtt - ca->curr_rtt;
else
jitter = ca->curr_rtt - ca->last_rtt;
sent = tp->snd_nxt - ca->interval_snd_seq_start;
received = tp->snd_una - ca->interval_una_seq_start;
sent_more_than_acked = (s64)(s32)(sent - received) >
(s64)div_u64((u64)tcp_snd_cwnd(tp) *
(u64)tp->mss_cache, 100);
if (!time_between32(now, ca->next_srrtt_check_ts
- 2 * ROCCET_MAX_SRRTT_CHECK_INTERVAL_MS
* USEC_PER_MSEC, ca->next_srrtt_check_ts)) {
evaluate_srrtt = true;
roccet_set_next_srrtt_check_ts(ca, now);
ca->interval_snd_seq_start = tp->snd_nxt;
ca->interval_una_seq_start = tp->snd_una;
}
if (tcp_in_cwnd_reduction(sk))
return;
roccet_xj = div_u64((u64)jitter * 100, ca->curr_min_rtt) +
sr_rtt_upper_bound;
if (ca->curr_srrtt > roccet_xj && evaluate_srrtt &&
sent_more_than_acked) {
roccet_congestion_event(sk, now);
return;
}
if (!tcp_is_cwnd_limited(sk) || sent_more_than_acked)
return;
if (ca->ack_carry_over) {
acked += ca->ack_carry_over;
ca->ack_carry_over = 0;
}
bictcp_update(ca, tcp_snd_cwnd(tp), acked);
tcp_cong_avoid_ai(tp, max(1, ca->cnt), acked);
}
static u32 roccet_recalc_ssthresh(struct sock *sk)
{
const struct tcp_sock *tp = tcp_sk(sk);
struct roccettcp *ca = inet_csk_ca(sk);
u32 cwnd = tcp_snd_cwnd(tp);
if (ca->state == LAUNCH)
return cwnd;
if (ca->state == RTT_PROBE)
cwnd = ca->cwnd_before_min_rtt_probe;
ca->epoch_start = 0;
if (cwnd < ca->last_max_cwnd && fast_convergence)
ca->last_max_cwnd = (cwnd * (BICTCP_BETA_SCALE + beta))
/ (2 * BICTCP_BETA_SCALE);
else
ca->last_max_cwnd = cwnd;
return max((cwnd * beta) / BICTCP_BETA_SCALE, 2U);
}
static u32 roccet_handle_recovery(struct sock *sk)
{
const struct tcp_sock *tp = tcp_sk(sk);
struct roccettcp *ca = inet_csk_ca(sk);
u32 cwnd = tcp_snd_cwnd(tp);
if (ca->state == RTT_PROBE_REFILL)
ca->state = ORBITER;
if (ca->state == RTT_PROBE) {
roccet_min_rtt_probe_ce(ca, cwnd);
return tcp_snd_cwnd(tp);
}
if (ca->state == LAUNCH) {
ca->state = ORBITER;
return cwnd;
}
return max((cwnd * beta) / BICTCP_BETA_SCALE, 2U);
}
static void roccet_handle_state_transitions(struct sock *sk,
enum roccet_state prev_state,
u32 now)
{
struct roccettcp *ca = inet_csk_ca(sk);
struct tcp_sock *tp = tcp_sk(sk);
u32 now_jiffies = tcp_jiffies32;
if (ca->state == prev_state)
return;
if (prev_state == LAUNCH) {
ca->is_in_initial_launch = false;
ca->initial_limit_reached = true;
}
if (ca->state == ORBITER) {
roccet_set_next_srrtt_check_ts(ca, now);
ca->interval_snd_seq_start = tp->snd_nxt;
ca->interval_una_seq_start = tp->snd_una;
if ((prev_state == RTT_PROBE ||
prev_state == RTT_PROBE_REFILL)) {
if (ca->epoch_start != 0) {
ca->epoch_start += now_jiffies -
ca->next_min_rtt_probe;
if (after(ca->epoch_start, now_jiffies))
ca->epoch_start = now_jiffies;
}
roccet_set_next_min_rtt_probe_ts(ca);
}
}
if (ca->state == DRAIN && prev_state == RTT_PROBE_REFILL)
roccet_set_next_min_rtt_probe_ts(ca);
if ((prev_state == RTT_PROBE_ENTER || prev_state == RTT_PROBE ||
prev_state == RTT_PROBE_REFILL) &&
(ca->state != RTT_PROBE && ca->state != RTT_PROBE_REFILL)) {
ca->probe_min_rtt_until = 0;
ca->refill_until = 0;
ca->rtt_probe_timers_set = false;
}
}
static void roccet_cwnd_event_tx_start(struct sock *sk)
{
struct roccettcp *ca = inet_csk_ca(sk);
struct tcp_sock *tp = tcp_sk(sk);
u32 now = tcp_jiffies32;
enum roccet_state prev_state = ca->state;
s32 delta;
delta = now - tcp_sk(sk)->lsndtime;
if (ca->epoch_start && delta > 0) {
ca->epoch_start += delta;
if (after(ca->epoch_start, now))
ca->epoch_start = now;
}
if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) &&
delta >= inet_csk(sk)->icsk_rto) {
tp->snd_ssthresh = tcp_snd_cwnd(tp);
tcp_snd_cwnd_set(tp, min(TCP_INIT_CWND, tp->snd_cwnd_clamp));
ca->state = LAUNCH;
}
roccet_handle_state_transitions(sk, prev_state, jiffies_to_usecs(now));
}
static void roccet_state(struct sock *sk, u8 new_state)
{
struct roccettcp *ca = inet_csk_ca(sk);
struct tcp_sock *tp = tcp_sk(sk);
u32 cwnd;
u32 now = jiffies_to_usecs(tcp_jiffies32);
enum roccet_state prev_state = ca->state;
if (new_state == TCP_CA_Loss) {
roccet_reset(sk, ca);
tcp_snd_cwnd_set(tp, min(TCP_INIT_CWND, tp->snd_cwnd_clamp));
WRITE_ONCE(tp->snd_ssthresh, max(tp->snd_ssthresh,
tcp_snd_cwnd(tp)));
} else if (tcp_in_cwnd_reduction(sk)) {
} else if (new_state == TCP_CA_CWR) {
cwnd = tcp_snd_cwnd(tp);
roccet_congestion_event(sk, now);
if (ca->state != RTT_PROBE_ENTER && ca->state != RTT_PROBE &&
ca->state != RTT_PROBE_REFILL)
ca->last_max_cwnd = cwnd;
} else if (new_state == TCP_CA_Recovery) {
cwnd = roccet_handle_recovery(sk);
tcp_snd_cwnd_set(tp, min(cwnd, tp->snd_cwnd_clamp));
}
roccet_handle_state_transitions(sk, prev_state, now);
}
static void roccet_acked(struct sock *sk, const struct ack_sample *sample)
{
struct roccettcp *ca = inet_csk_ca(sk);
bool after_fast_recovery;
u32 delay;
if (sample->rtt_us < 0)
return;
after_fast_recovery = ca->epoch_start &&
(s32)(tcp_jiffies32 - ca->epoch_start) < HZ;
if (!after_fast_recovery || ca->state == RTT_PROBE) {
if (sample->rtt_us > 0) {
ca->last_rtt = ca->curr_rtt;
ca->curr_rtt = sample->rtt_us;
}
}
if (after_fast_recovery)
return;
delay = sample->rtt_us;
if (delay == 0)
delay = 1;
if (ca->delay_min == 0 || (s32)(delay - ca->delay_min) < 0)
ca->delay_min = delay;
}
static void roccet_update_pacing_rate(struct sock *sk)
{
struct roccettcp *ca = inet_csk_ca(sk);
struct tcp_sock *tp = tcp_sk(sk);
u64 rate;
rate = (u64)tp->mss_cache * ((div64_u64(USEC_PER_SEC, 100) << 3));
if (ca->state == RTT_PROBE || tcp_snd_cwnd(tp) >= tp->snd_ssthresh / 2)
rate *= 100;
else
rate *= READ_ONCE
(sock_net(sk)->ipv4.sysctl_tcp_pacing_ss_ratio);
rate *= max(tcp_snd_cwnd(tp), tp->packets_out);
if (likely(tp->srtt_us))
do_div(rate, tp->srtt_us);
WRITE_ONCE(sk->sk_pacing_rate,
min_t(u64, rate, READ_ONCE(sk->sk_max_pacing_rate)));
}
static void roccet_drain_update(struct sock *sk, u32 now)
{
struct tcp_sock *tp = tcp_sk(sk);
struct roccettcp *ca = inet_csk_ca(sk);
if (!time_between32(now, ca->roccet_last_event_time_us
- 100 * USEC_PER_MSEC,
ca->roccet_last_event_time_us
+ 100 * USEC_PER_MSEC)) {
if (tcp_in_slow_start(tp))
ca->state = LAUNCH;
else
ca->state = ORBITER;
}
}
static void roccet_control(struct sock *sk, u32 ack, int flag,
const struct rate_sample *rs)
{
struct roccettcp *ca = inet_csk_ca(sk);
u32 now = jiffies_to_usecs(tcp_jiffies32);
enum roccet_state prev_state = ca->state;
update_ack_rate(ca, rs->acked_sacked, now);
if (rs->acked_sacked > 0) {
update_min_rtt(ca);
update_srrtt(ca);
}
switch (ca->state) {
case LAUNCH:
roccet_launch_update(sk, rs->acked_sacked);
break;
case ORBITER:
roccet_orbiter_update(sk, rs->acked_sacked);
break;
case DRAIN:
roccet_drain_update(sk, now);
break;
case RTT_PROBE_ENTER:
roccet_enter_min_rtt_probe(sk, now);
break;
case RTT_PROBE:
roccet_min_rtt_probe(sk, now);
break;
case RTT_PROBE_REFILL:
roccet_rtt_probe_refill(ca, now);
break;
default:
pr_err_once("TCP ROCCET: Invalid state %d\n", ca->state);
}
roccet_handle_state_transitions(sk, prev_state, now);
roccet_update_pacing_rate(sk);
if (!ca->initial_limit_reached &&
(tcp_is_cwnd_limited(sk) ||
!time_between32(now, ca->roccet_last_event_time_us
- 500 * USEC_PER_MSEC,
ca->roccet_last_event_time_us
+ 500 * USEC_PER_MSEC)))
ca->initial_limit_reached = true;
}
static struct tcp_congestion_ops roccet_tcp __read_mostly = {
.init = roccet_init,
.ssthresh = roccet_recalc_ssthresh,
.set_state = roccet_state,
.undo_cwnd = tcp_reno_undo_cwnd,
.cwnd_event_tx_start = roccet_cwnd_event_tx_start,
.pkts_acked = roccet_acked,
.cong_control = roccet_control,
.owner = THIS_MODULE,
.name = "roccet",
};
static int __init roccet_register(void)
{
int param_err;
BUILD_BUG_ON(sizeof(struct roccettcp) > ICSK_CA_PRIV_SIZE);
param_err = param_check();
if (param_err)
return param_err;
param_precompute();
return tcp_register_congestion_control(&roccet_tcp);
}
static void __exit roccet_unregister(void)
{
tcp_unregister_congestion_control(&roccet_tcp);
}
module_init(roccet_register);
module_exit(roccet_unregister);
MODULE_AUTHOR("Lukas Prause, Tim Füchsel");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("ROCCET TCP"