bandwidthTool is a tiny program to measure the end to end capacity (bottleneck bandwidth) of a path with the help of the packet train dispersion technique. The main idea is to send multiple packet pairs/trains from one end to the other consisting of multiple packets of the same size sent back-to-back. Then, the dispersion (distance between the last bit of the first and last packet) is measured, corresponding to the capacity if the links do not carry other traffic (see figure). Cross-traffic can increase or decrease the measured dispersion, causing under- or overestimation of the path capacity.

Measurements can be adapted by either changing the packet size or using different train lengths. Using the default settings below, 800 * 15 bytes (11.7 KiB) are sufficient for one single measurement.

/*
 *      bandwidthTool
 *      end to end capacity (bottleneck bandwidth) measurement
 *      with the help of the packet train dispersion technique
 *      
 *      Copyright (C) 2010 Alexander Heinlein
 *      
 *      This program is free software; you can redistribute it and/or modify
 *      it under the terms of the GNU General Public License as published by
 *      the Free Software Foundation; either version 3 of the License, or
 *      (at your option) any later version.
 *      
 *      This program is distributed in the hope that it will be useful,
 *      but WITHOUT ANY WARRANTY; without even the implied warranty of
 *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *      GNU General Public License for more details.
 *      
 *      You should have received a copy of the GNU General Public License
 *      along with this program; if not, write to the Free Software
 *      Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 *      MA 02110-1301, USA.
 */
 
#include <iostream>
#include <vector>
#include <cassert>
#include <string>
#include <cstring>
 
#include <sys/socket.h>
#include <sys/time.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <netdb.h>
#include <errno.h>
 
#define UDP_PORT 15000
#define TCP_PORT 15000
#define RTT_PROBES 10
// size of each packet in bytes
#define PACKET_LEN 800
// number of packets per train
#define TRAIN_LEN 15
 
/// time difference in micro seconds
int SOBandwidthTool::timeDiff(const struct timeval& tv1, const struct timeval& tv2) {
    return (tv2.tv_sec - tv1.tv_sec) * 1000 * 1000 + (tv2.tv_usec - tv1.tv_usec);
}
 
int SOBandwidthTool::receiver(const std::string& hostAddr) {  
    // get host address list
    struct addrinfo hints;
    memset(&hints, 0, sizeof(struct addrinfo));
    hints.ai_socktype = SOCK_STREAM;
    std::stringstream cport;
    cport << TCP_PORT;
    struct addrinfo* addrList;
    int err = getaddrinfo(hostAddr.c_str(), cport.str().c_str(), &hints, &addrList);
    if(err != 0) {
        std::cerr << "Error: Could not resolve host address: " << gai_strerror(err) << std::endl;
        return 1;
    }
 
    // initialize tcp socket (for rtt measurements)
    int tcpSock;
    if((tcpSock = socket(AF_INET, SOCK_STREAM, 0)) == -1) {
        std::cerr << "Error: Could not create TCP socket: " << strerror(errno) << std::endl;
        return -1;
    }
 
    int opt = 1;
    if(setsockopt(tcpSock, SOL_SOCKET, SO_REUSEADDR, (char*)&opt, sizeof(opt)) == -1) {
        std::cerr << "Error: Could not set SO_REUSEADDR on TCP socket: " << strerror(errno) << std::endl;
        close(tcpSock);
        return -1;
    }
 
    // getaddrinfo() returns a list of addresses, try each until connect() succeeds
    struct addrinfo* addr;
    for(addr = addrList; addr != NULL; addr = addr->ai_next) {
        std::cout << "Trying to connect to " << inet_ntoa(((struct sockaddr_in*)addr->ai_addr)->sin_addr)
                  << ":" << ntohs(((struct sockaddr_in*)addr->ai_addr)->sin_port) << std::endl;
        if(connect(tcpSock, addr->ai_addr, addr->ai_addrlen) != -1) {
            // got right address, keep pointer for udp socket later
            break;
        }
    }
 
    if(addr == NULL) {
        std::cerr << "Error: Could not connect to host " << hostAddr << std::endl;
        close(tcpSock);
        return -1;
    }
 
    //
    // calculate RTT
    //
    int totalRtt = 0;
    struct timeval now, last;
    int msg = 42;
    std::cout << "Measuring RTT: ";
    for(unsigned int i = 0; i < RTT_PROBES; i++) {
        // transmit message
        if(send(tcpSock, &msg, sizeof(msg), 0) != sizeof(msg)) {
            std::cerr << "Error: Could not send on TCP socket: " << strerror(errno) << std::endl;
            close(tcpSock);
            return -1;
        }
        gettimeofday(&last, 0);
 
        // receive reply
        char buf[1024];
        if(recv(tcpSock, buf, sizeof(msg), 0) != sizeof(msg)) {
            std::cerr << "Error: Could not receive on TCP socket: " << strerror(errno) << std::endl;
            close(tcpSock);
            return -1;
        }
        gettimeofday(&now, 0);
 
        if(i > 1) {  // skip first two results
            int duration = timeDiff(last, now);
            totalRtt += duration;
            std::cout << duration / 1000.0 << " ms  " << std::flush;
        }
    }
    std::cout << std::endl;
    close(tcpSock);
 
    double rtt = totalRtt / ((RTT_PROBES - 2) * 1000.0);
    std::cout << "Average RTT is " << rtt << " ms" << std::endl;
 
    //
    // send packet pairs/trains to measure dispersion
    //
    std::cout << "Measuring dispersion." << std::endl;
 
    // initialize udp socket (for packet dispersion measurements)
    int udpSock;
    if((udpSock = socket(AF_INET, SOCK_DGRAM, 0)) == -1) {
        std::cerr << "Error: Could not create UDP socket: " << strerror(errno) << std::endl;
        return -1;
    }
 
    struct sockaddr_in udpAddr;
    memset(&udpAddr, 0, sizeof udpAddr);
    udpAddr.sin_family      = AF_INET;
    udpAddr.sin_addr.s_addr = ((struct sockaddr_in*)addr->ai_addr)->sin_addr.s_addr;
    udpAddr.sin_port        = htons(UDP_PORT);
 
    freeaddrinfo(addr);
 
    std::string packet;
    packet.resize(PACKET_LEN - 1, '.');
    assert(packet.length() + 1 == PACKET_LEN);
 
    std::cout << "...sending packet ";
    for(unsigned int i = 0; i < TRAIN_LEN; i++) {
        std::cout << i << " " << std::flush;
        if(sendto(udpSock, packet.c_str(), PACKET_LEN, 0, (struct sockaddr*)&udpAddr, sizeof(udpAddr)) != PACKET_LEN) {
            std::cerr << "Error: Could not send on UDP socket: " << strerror(errno) << std::endl;
            close(udpSock);
            return -1;
        }
    }
    std::cout << std::endl;
    close(udpSock);
 
    return 0;     
}
 
int SOBandwidthTool::sender() {
    // initialize tcp socket (for rtt measurements)
    int tcpSock;
    if((tcpSock = socket(AF_INET, SOCK_STREAM, 0)) == -1) {
        std::cerr << "Error: Could not create TCP socket: " << strerror(errno) << std::endl;
        return -1;
    }
 
    int opt = 1;
    if(setsockopt(tcpSock, SOL_SOCKET, SO_REUSEADDR, (char*)&opt, sizeof(opt)) == -1) {
        std::cerr << "Error: Could not set SO_REUSEADDR on TCP socket: " << strerror(errno) << std::endl;
        close(tcpSock);
        return -1;
    }
 
    struct sockaddr_in tcpAddr;
    tcpAddr.sin_family      = AF_INET;
    tcpAddr.sin_addr.s_addr = htonl(INADDR_ANY);
    tcpAddr.sin_port        = htons(TCP_PORT);
 
    if(bind(tcpSock, (struct sockaddr*)&tcpAddr, sizeof(tcpAddr)) == -1) {
        std::cerr << "Error: Could not bind TCP socket: " << strerror(errno) << std::endl;
        close(tcpSock);
        return -1;
    }
 
    std::cout << "Listening on TCP socket on port " << TCP_PORT << std::endl;
    if(listen(tcpSock, 1) == -1) {
        std::cerr << "Error: Could not listen on TCP socket: " << strerror(errno) << std::endl;
        close(tcpSock);
        return -1;
    }
 
    int conSock;
    socklen_t tcpAddrLen = sizeof(tcpAddr);
    if((conSock = accept(tcpSock, (struct sockaddr*)&tcpAddr, &tcpAddrLen)) == -1) {
        std::cerr << "Error: Could not accept TCP connection: " << strerror(errno) << std::endl;
        close(tcpSock);
        return -1;
    }
    std::cout << "Incoming connection from " << inet_ntoa(tcpAddr.sin_addr) << std::endl;
 
    //
    // reply to RTT measurements
    //
    int msg = 42;
    std::cout << "Measuring RTT." << std::endl;
    for(unsigned int i = 0; i < RTT_PROBES; i++) {
        char buf[1024];
        if(recv(conSock, buf, sizeof(msg), 0) != sizeof(msg)) {
            std::cerr << "Error: Could not receive on TCP socket: " << strerror(errno) << std::endl;
            close(tcpSock);
            return -1;
        }
 
        if(send(conSock, &msg, sizeof(msg), 0) != sizeof(msg)) {
            std::cerr << "Error: Could not send on TCP socket: " << strerror(errno) << std::endl;
            close(tcpSock);
            return -1;
        }
    }
    close(tcpSock);
 
    //
    // receive packet pairs/trains to measure dispersion
    //
    std::cout << "Measuring dispersion." << std::endl;
 
    // initialize udp socket
    int udpSock;
    if((udpSock = socket(AF_INET, SOCK_DGRAM, 0)) == -1) {
        std::cerr << "Error: Could not create UDP socket: " << strerror(errno) << std::endl;
        return -1;
    }
 
    struct sockaddr_in udpAddr;
    memset(&udpAddr, 0, sizeof udpAddr);
    udpAddr.sin_family      = AF_INET;
    udpAddr.sin_addr.s_addr = htonl(INADDR_ANY);
    udpAddr.sin_port        = htons(UDP_PORT);
 
    if(bind(udpSock, (struct sockaddr*)&udpAddr, sizeof(udpAddr)) == -1) {
        std::cerr << "Error: Could not bind UDP socket: " << strerror(errno) << std::endl;
        close(udpSock);
        return -1;
    }
 
    socklen_t udpAddrLen = sizeof(udpAddr);
    struct timeval first, last;
    char recvMsg[PACKET_LEN];
 
    std::cout << "...waiting for packet ";
    for(unsigned int i = 0; i < TRAIN_LEN; i++) {
        std::cout << i << " " << std::flush;
        if(recvfrom(udpSock, recvMsg, PACKET_LEN, 0, (struct sockaddr*)&udpAddr, &udpAddrLen) != PACKET_LEN) {
            std::cerr << "Error: Could not receive on UDP socket: " << strerror(errno) << std::endl;
            close(udpSock);
            return -1;
        }
 
        if(i == 0) {
            gettimeofday(&first, 0);
        }
    }
    gettimeofday(&last, 0);
    std::cout << std::endl;
    close(udpSock);
 
    int    diff_ns = timeDiff(first, last) * 1000;
    double diff_s  = timeDiff(first, last) / 1000.0 / 1000.0;
    std::cout << "Duration: "
              << diff_ns << " ns = "
              << diff_ns / 1000.0 / 1000.0 << " ms" << std::endl;
 
    double disp_s = ((TRAIN_LEN - 1) * PACKET_LEN) / double(diff_s);
    std::cout << "Dispersion: " << disp_s / 1000.0 << " ms" << std::endl;
    std::cout << "Capacity: "
              << disp_s * 8.0 << " bit/s = "
              << disp_s * 8.0 / 1000.0 << " kbit/s = "
              << disp_s * 8.0 / 1000.0 / 1000.0 << " Mbit/s = "
              << disp_s * 8.0 / 1000.0 / 1000.0 / 1000.0 << " Gbit/s" << std::endl;
 
    return 0;
}
 
void usage(const std::string& cmd) {
    std::cout << std::endl
              << "Usage: " << cmd << " [OPTION]" << std::endl
              << "-s, --send           act as sender" << std::endl
              << "-r, --receive ADDR   act as receiver" << std::endl
              << "-h, --help           print this help and exit" << std::endl;
}
 
int main(int argc, char* argv[]) {
    if(argc < 2) {
        std::cerr << "You don't like arguments? So i don't like you :(" << std::endl;
        usage(argv[0]);
        return 0;
    }
 
    SOBandwidthTool bw;
 
    std::vector<std::string> args(argv, argv + argc);
    for(std::vector<std::string>::iterator it = args.begin() + 1; it < args.end(); it++) {
        if(*it == "-s" || *it == "--send") {
            bw.sender();
        } else if(*it == "-r" || *it == "--receive") {
            if(++it < args.end()) {
                bw.receiver(*it);
            } else {
                std::cerr << "Error: No address specified." << std::endl;
                usage(argv[0]);
                return -1;
            }
        } else if(*it == "-h" || *it == "--help") {
            usage(argv[0]);
            return 0;
        } else {
            std::cerr << "Error: Invalid argument: " << *it << std::endl;
            usage(argv[0]);
            return -1;
        }
    }
 
    return 0;
}

To analyze the capacity estimation method, different scenarios were tested with 100 measurements for each of them.

• capacity: the capacity of the bottleneck link (there may be other links with higher capacity on the path)
• cross-traffic: other traffic on the path

As this table shows, cross-traffic can influence measurements heavily (especially in the 100 Mbit scenario). Yet, the median is stable enough to rely on it, providing no overestimation of the capacity.

Here, (even light) cross-traffic leads to significant measurement errors. Nevertheless the median is quite stable and provides an adequate capacity estimation.

Note: As there is no constant medium using wireless networks, the capacity was measured using iperf.

The following figures illustrate the measured capacities (100 tests each). Also note the y-axis when comparing two similar looking plots.

192kbit:

2Mbit: 2Mbit (cross-traffic):

100Mbit: 100Mbit (cross-traffic):

24Mbit: 24Mbit (cross-traffic):

• Bandwidth estimation: metrics, measurement techniques, and tools
• What do packet dispersion techniques measure?
• Packet dispersion techniques and a capacity estimation methodology