test_harness.cc

A 'kitchen sink' program that will exercise much of the library's functionality.

#include <iostream>             // cout, cerr
#include <string>

#include "data_structures/basic_types.h"        // Timeval
#include "data_structures/PacketFilter.h"
#include "data_structures/Aggregate.h"          // Trace
#include "protocols/parsers/QcapAdaptor.h"
#include "data_structures/Packet.h"
#include "protocols/parsers/IPv4Demux.h"
#include "protocols/parsers/StreamedTCPParser.h"
#include "protocols/parsers/StreamedUDPParser.h"
#include "attributes/ByteCountAttribute.h"
#include "attributes/HistogramAttribute.h"

#include "protocols/datatypes/IPv4Filters.h"
#include "protocols/datatypes/TCPFilters.h"

#include <sys/times.h>          // CPU times for profiling

using namespace std; 

int usage( void ) {
    cerr << "Usage:  test_harness <pcap>" << endl;
    return 0;
}

int main( int argc, char **argv ) {
    int printout_packets = 0;
    int printout_conversations = 0;
    int printout_histograms = 1;
    int profile = 1;

    struct tms checkpoint;
    struct tms tms;

    if( ( argc < 2 ) || ( argc > 2 ) ) {
        usage();
        exit(1);
    }

    if( profile ) {
        times(&checkpoint);
        cerr << "Profiling started" << endl;
    }
    
    // basic attributes
    PacketAttribute timestampAttrib( "frame.time" );
    PacketAttribute capLenAttrib( "frame.cap_len" );
    PacketAttribute pktLenAttrib( "frame.pkt_len" );

    // IP attributes
    PacketAttribute ipSrcAttrib( "ip.src" );
    PacketAttribute ipDstAttrib( "ip.dst" );
    PacketAttribute ipProtoAttrib( "ip.proto" );

    // TCP attributes
    PacketAttribute tcpSrcAttrib( "tcp.srcport" );
    PacketAttribute tcpDstAttrib( "tcp.dstport" );

    // prep a TCP filter
    const PacketFilter *const tcpFilter = IPv4Filters::IPV4_PROTO_TCP;

    // prep a UDP filter
    const PacketFilter *const udpFilter = IPv4Filters::IPV4_PROTO_UDP;

    // prep a ICMP filter
    PacketFilter *icmpFilter = NULL;
    icmpFilter = new SimplePacketFilter( 
            PacketAttribute( "ip.proto" ), 
            ConcretePacketAttributeValue<int>(1),
            SimplePacketFilter::EQUAL );

    // prep a compound filter for ozymandias
    PacketFilter *ozyFilter = new CompoundPacketFilter( 
            SimplePacketFilter(
                PacketAttribute( "ip.src" ),
                ConcretePacketAttributeValue<string>( "192.168.1.2" ),
                SimplePacketFilter::EQUAL ),
            SimplePacketFilter(
                PacketAttribute( "ip.dst" ),
                ConcretePacketAttributeValue<string>( "192.168.1.2" ),
                SimplePacketFilter::EQUAL ),
            CompoundPacketFilter::OR );

    // prep a compound filter for ozymandias TCP
    PacketFilter *ozyTcpFilter = NULL;
    ozyTcpFilter = new CompoundPacketFilter(
            *tcpFilter,
            *ozyFilter,
            CompoundPacketFilter::AND );

    // prep a compound filter for shakespeare
    PacketFilter *shakespeareFilter = new CompoundPacketFilter( 
            SimplePacketFilter(
                PacketAttribute( "ip.src" ),
                ConcretePacketAttributeValue<string>( "134.117.225.11" ),
                SimplePacketFilter::EQUAL ),
            SimplePacketFilter(
                PacketAttribute( "ip.dst" ),
                ConcretePacketAttributeValue<string>( "134.117.225.11" ),
                SimplePacketFilter::EQUAL ),
            CompoundPacketFilter::OR );

    // shakespeare udp
    PacketFilter *shakespeareUdpFilter = NULL;
    shakespeareUdpFilter = new CompoundPacketFilter(
            *shakespeareFilter,
            *udpFilter,
            CompoundPacketFilter::AND );

    // TCP port 22
    const PacketFilter *const sshFilter = TCPFilters::TCP_PORT_22_SSH;

    // TCP port 22 - StreamKey filter
    const PacketFilter *const sshStreamFilter = TCPFilters::TCP_PORT_22_SSH_STREAM;

    // TCP port 80
    const PacketFilter *const wwwFilter = TCPFilters::TCP_PORT_80_WWW;

    // TCP port 80 - StreamKey filter
    const PacketFilter *const wwwStreamFilter = TCPFilters::TCP_PORT_80_WWW_STREAM;

    // TCP port 993
    const PacketFilter *const imapsFilter = TCPFilters::TCP_PORT_993_IMAPS;

    // TCP port 993 - StreamKey filter
    const PacketFilter *const imapsStreamFilter = TCPFilters::TCP_PORT_993_IMAPS_STREAM;

    if( profile ) {
        times(&tms);
        cerr << "Built filters : " 
            << ( tms.tms_utime - checkpoint.tms_utime ) << " user, "
            << ( tms.tms_stime - checkpoint.tms_stime ) << " sys" << endl;
        checkpoint = tms;
    }

    // set up the parsing infrastructure

    string filename( argv[1] );

    // create an IP trace and populate it with the contents of the pcap file
    Trace               ethTrace;       // ethernet packets
    Trace               ipFragTrace;    // IPv4 fragments
    Trace               ipTrace;        // non-fragmented and reassembled IPv4 packets
    Decomposition       ipDemuxed;      // IPv4 packets demuxed by (src,dst,proto)
    Decomposition       tcpSessions;    // TCP sessions
    Decomposition       udpSessions;    // UDP sessions

    ByteCountAttribute  bytecount;
    ByteCountAttribute  bytecountPayload(false);

    // add histograms to individual TCP streams, and for ssh, www, and imaps streams
    int lengthBounds[23] = {
           0,    1,    2,    4,    5, 
           9,   19,   39,   49,   99, 
         179,  235,  268,  349,  449, 
         515,  548,  649,  999, 1379, 
        1380, 1431, 1472 
    };
    Timeval delayBounds[8] = { 
        Timeval( 0.000001 ), Timeval( 0.0001 ), Timeval( 0.001 ), Timeval( 0.01 ), Timeval( 0.1 ),
             Timeval( 1.0 ),   Timeval( 10.0 ),  Timeval( 64.0 )
    };

    // set up the decompositions with bytecount attributes
    
    // set up the IPv4 demultiplexer
    Trace *ipTemplate = new Trace();
    ipTemplate->addAttrib( "bytecount", bytecount );
    IPv4Demux ipDemuxer( ipDemuxed, ipTemplate );
    ipTrace.addListener( ipDemuxer );

    Trace *tcpTemplate = new Trace();
    tcpTemplate->addAttrib( "bytecount", bytecount );
    tcpTemplate->addAttrib( "bytecount_payload", bytecountPayload );
    // per-stream length and delay histograms
    tcpTemplate->addAttrib( "length_hist", HistogramAttribute<int>( PacketAttribute( "tcp.len" ), lengthBounds, 23 ) );
    tcpTemplate->addAttrib( "delay_hist", HistogramAttribute<Timeval>( PacketAttribute( "frame.time_delta" ), delayBounds, 8 ) );

    // packet counts and length and delay histograms for all packets from particular applications
    tcpSessions.addAttrib( "ssh_pkt_count", PacketCountAttribute( *sshFilter ) );
    tcpSessions.addAttrib( "ssh_length_hist", HistogramAttribute<int>( *sshFilter, PacketAttribute( "tcp.len" ), lengthBounds, 23 ) );
    tcpSessions.addAttrib( "ssh_delay_hist", HistogramAttribute<Timeval>( *sshFilter, PacketAttribute( "frame.time_delta" ), delayBounds, 8 ) );
    tcpSessions.addAttrib( "www_pkt_count", PacketCountAttribute( *wwwFilter ) );
    tcpSessions.addAttrib( "www_length_hist", HistogramAttribute<int>( *wwwFilter, PacketAttribute( "tcp.len" ), lengthBounds, 23 ) );
    tcpSessions.addAttrib( "www_delay_hist", HistogramAttribute<Timeval>( *wwwFilter, PacketAttribute( "frame.time_delta" ), delayBounds, 8 ) );
    tcpSessions.addAttrib( "imaps_pkt_count", PacketCountAttribute( *imapsFilter ) );
    tcpSessions.addAttrib( "imaps_length_hist", HistogramAttribute<int>( *imapsFilter, PacketAttribute( "tcp.len" ), lengthBounds, 23 ) );
    tcpSessions.addAttrib( "imaps_delay_hist", HistogramAttribute<Timeval>( *imapsFilter, PacketAttribute( "frame.time_delta" ), delayBounds, 8 ) );

    // set up the TCP parser
    StreamedTCPParser tcpParser( tcpSessions, tcpTemplate );
    ipDemuxed.addListener( tcpParser );

    // set up the UDP parser
    Trace *udpTemplate = new Trace();
    udpTemplate->addAttrib( "bytecount", bytecount );

    StreamedUDPParser udpParser( udpSessions, udpTemplate );
    ipDemuxed.addListener( udpParser );

    if( profile ) {
        times(&tms);
        cerr << "Initialized aggregates : " 
            << ( tms.tms_utime - checkpoint.tms_utime ) << " user, "
            << ( tms.tms_stime - checkpoint.tms_stime ) << " sys" << endl;
        checkpoint = tms;
    }

    // process traffic
    QcapAdaptor qcap(filename);

    // start reading packets - demultiplexing and sessionizing will be done as packets are added to
    // ipTrace
    qcap.getIPTrace( ethTrace, ipFragTrace, ipTrace );

    if( profile ) {
        times(&tms);
        cerr << "Built IPv4 trace : " 
            << ( tms.tms_utime - checkpoint.tms_utime ) << " user, "
            << ( tms.tms_stime - checkpoint.tms_stime ) << " sys" << endl;
        checkpoint = tms;
    }

    TraceIterator it = ethTrace.begin();
    TraceIterator end = ethTrace.end();

    if( printout_packets ) {
        int count = 0;
        while( it != end ) {
            // cout << "(a) " << it.toString() << endl;
            try {
                // cout << "(b) ";
                cout << (*it).getAttrib(timestampAttrib) << ", ";
                cout << (*it).getAttrib(capLenAttrib) << "/" ;
                cout << (*it).getAttrib(pktLenAttrib) << " bytes";
            } catch( runtime_error err ) {
                cerr << "Error outputting packet attributes" << endl;
            }

            try {
                cout << " : proto " << (*it).getAttrib(ipProtoAttrib) << ", ";
                cout << (*it).getAttrib(ipSrcAttrib) << " -> " ;
                cout << (*it).getAttrib(ipDstAttrib);
            } catch( UnknownAttributeException unkAtt ) { 
                // ignore, expected for non-IP pkts
            }

            if( tcpFilter->match(it) ) {
                cout << " TCP";
            }

            if( udpFilter->match(it) ) {
                cout << " UDP";
            }

            if( icmpFilter->match(it) ) {
                cout << " ICMP";
            }

            if( shakespeareUdpFilter->match(it) ) {
                cout << " SHAKESPEARE_UDP";
            }

            if( ozyTcpFilter->match(it) ) {
                cout << " OZYMANDIAS_TCP";
            }

            cout << endl;

            count++;
            ++it;
        }

        cout << count << " packets." << endl;
    }

    // print out the count of packets
    cout << ethTrace.getAttrib("count") << " packets in trace." << endl;

    if( profile ) {
        times(&tms);
        cerr << "Iterated through trace : " 
            << ( tms.tms_utime - checkpoint.tms_utime ) << " user, "
            << ( tms.tms_stime - checkpoint.tms_stime ) << " sys" << endl;
        checkpoint = tms;
    }
    
    // testAttribValues();

    // print out the count of TCP packets
    //cout << "TCP packets: " << tcpSessions.getAttrib("count") << endl;
    if( printout_conversations ) {
        map<StreamKeyWrapper, Aggregate*>::const_iterator decompIt = ipDemuxed.substreamsBegin();

        cout << "== IPv4 conversations ==" << endl;
        while( decompIt != ipDemuxed.substreamsEnd() ) {
            Trace *ipv4Stream = dynamic_cast<Trace*>(decompIt->second);
            cout << "** " << ipv4Stream->getStreamKey().toString() 
                << ", " << ipv4Stream->getAttrib("count") << " packets" 
                << ", " << ipv4Stream->getAttrib("bytecount") << " bytes"
                << endl;
            ++decompIt;
        }

        if( profile ) {
            times(&tms);
            cerr << "Iterated through IPv4 streams : " 
                << ( tms.tms_utime - checkpoint.tms_utime ) << " user, "
                << ( tms.tms_stime - checkpoint.tms_stime ) << " sys" << endl;
            checkpoint = tms;
        }
    }

    if( printout_conversations ) {
        map<StreamKeyWrapper, Aggregate*>::const_iterator tcpIt = tcpSessions.substreamsBegin();

        cout << "== TCP conversations ==" << endl;

        while( tcpIt != tcpSessions.substreamsEnd() ) {
            Trace *tcpStream = dynamic_cast<Trace*>(tcpIt->second);
            cout << "*** " << tcpStream->getStreamKey().toString() 
                << ", " << tcpStream->getAttrib("count") << " packets" 
                << ", " << tcpStream->getAttrib("bytecount") << " bytes"
                << ", " << tcpStream->getAttrib("bytecount_payload") << " bytes payload"
                << endl;

            ++tcpIt;
        }

        // print out the count of TCP packets
        cout << "TCP packets: " << tcpSessions.getAttrib("count") << endl;

        if( profile ) {
            times(&tms);
            cerr << "Iterated through TCP sessions : " 
                << ( tms.tms_utime - checkpoint.tms_utime ) << " user, "
                << ( tms.tms_stime - checkpoint.tms_stime ) << " sys" << endl;
            checkpoint = tms;
        }
    }

    if( printout_conversations ) {
        map<StreamKeyWrapper, Aggregate*>::const_iterator udpIt = udpSessions.substreamsBegin();

        cout << "== UDP conversations ==" << endl;

        while( udpIt != udpSessions.substreamsEnd() ) {
            Trace *udpStream = dynamic_cast<Trace*>(udpIt->second);
            cout << "*** " << udpStream->getStreamKey().toString() 
                << ", " << udpStream->getAttrib("count") << " packets" 
                << ", " << udpStream->getAttrib("bytecount") << " bytes"
                << endl;

            ++udpIt;
        }

        // print out the count of UDP packets
        cout << "UDP packets: " << udpSessions.getAttrib("count") << endl;

        if( profile ) {
            times(&tms);
            cerr << "Iterated through UDP sessions : " 
                << ( tms.tms_utime - checkpoint.tms_utime ) << " user, "
                << ( tms.tms_stime - checkpoint.tms_stime ) << " sys" << endl;
            checkpoint = tms;
        }
    }

    // print out the count of ICMP packets
    /*
    Aggregate *icmp = ethTrace.subset( *icmpFilter );

    if( icmp != NULL ) {
        cout << "ICMP packets: " << icmp->getAttrib("count") << endl;
        delete icmp;
    } else {
        cerr << "ethTrace.subset returned null for filter " << icmpFilter << endl;
    }

    if( profile ) {
        times(&tms);
        cerr << "Isolated ICMP packets : " 
            << ( tms.tms_utime - checkpoint.tms_utime ) << " user, "
            << ( tms.tms_stime - checkpoint.tms_stime ) << " sys" << endl;
        checkpoint = tms;
    }
    */

    /* Need to adjust this to deal with the fact that tcpSessions is a Decomposition
    Aggregate *ssh = tcpSessions.subset( *sshFilter );

    if( ssh != NULL ) {
        cout << "SSH packets: " << ssh->getAttrib("count") << endl;
        delete ssh;
    } else {
        cerr << "ethTrace.subset returned null for filter " << sshFilter << endl;
    }

    if( profile ) {
        times(&tms);
        cerr << "Isolated SSH packets : " 
            << ( tms.tms_utime - checkpoint.tms_utime ) << " user, "
            << ( tms.tms_stime - checkpoint.tms_stime ) << " sys" << endl;
        checkpoint = tms;
    }
    */

    // print out the number of unique IP addresses 
    // - note that there will probably be more than one value for ip.addr per packet, which will
    //   have to be handled gracefully
    //ethTrace.addAttrib( "uniqueIPAddresses", SetAttribute( PacketAttribute( "ip.addr" ) ) );
    //cout << "Number of unique IP addresses: " << ethTrace.getAttrib( "uniqueIPAddresses.count" ) << endl;
    
    // print out a histogram of inter-packet delays
    //ethTrace.addAttrib( "packetDelayHist", HistogramAttribute( PacketAttribute( "frame.time_delta" ) ) );
    //cout << "Packet delay histogram: " << ethTrace.getAttrib( "packetDelayHist" ) << endl;
 

    // print out histograms for ssh, www, and imaps
 
    if( printout_histograms ) {
        Decomposition::const_iterator tcpIt = tcpSessions.substreamsBegin();

        cout << "== SSH conversations ==" << endl;

        cout << tcpSessions.getAttrib("ssh_pkt_count") << " SSH packets" << endl;

        while( tcpIt != tcpSessions.substreamsEnd() ) {
            Trace *tcpStream = dynamic_cast<Trace*>(tcpIt->second);
            if( sshStreamFilter->match( tcpStream->getStreamKey() ) ) {
                cout << "*** " << tcpStream->getStreamKey().toString() 
                    << ", " << tcpStream->getAttrib("count") << " packets" 
                    << ", " << tcpStream->getAttrib("bytecount") << " bytes"
                    << ", " << tcpStream->getAttrib("bytecount_payload") << " bytes payload"
                    << endl;
                cout << "Length histogram: " << tcpStream->getAttrib("length_hist") << endl;
                cout << "Delay histogram: " << tcpStream->getAttrib("delay_hist") << endl;
            }

            ++tcpIt;
        }

        // print out the histogram of SSH packet lengths
        cout << tcpSessions.getAttrib("ssh_pkt_count") << " SSH packets" << endl;
        cout << "-- SSH TCP payload length histogram --" << endl;
        cout << tcpSessions.getAttrib("ssh_length_hist") << endl;
    }

    if( printout_histograms ) {
        Decomposition::const_iterator tcpIt = tcpSessions.substreamsBegin();

        cout << "== WWW conversations ==" << endl;

        cout << tcpSessions.getAttrib("www_pkt_count") << " WWW packets" << endl;

        while( tcpIt != tcpSessions.substreamsEnd() ) {
            Trace *tcpStream = dynamic_cast<Trace*>(tcpIt->second);
            if( wwwStreamFilter->match( tcpStream->getStreamKey() ) ) {
                cout << "*** " << tcpStream->getStreamKey().toString() 
                    << ", " << tcpStream->getAttrib("count") << " packets" 
                    << ", " << tcpStream->getAttrib("bytecount") << " bytes"
                    << ", " << tcpStream->getAttrib("bytecount_payload") << " bytes payload"
                    << endl;
                cout << "Length histogram: " << tcpStream->getAttrib("length_hist") << endl;
                cout << "Delay histogram: " << tcpStream->getAttrib("delay_hist") << endl;
            }

            ++tcpIt;
        }

        // print out the histogram of WWW packet lengths
        cout << "-- WWW TCP payload length histogram --" << endl;
        cout << tcpSessions.getAttrib("www_length_hist") << endl;

        if( profile ) {
            times(&tms);
            cerr << "Iterated through WWW sessions : " 
                << ( tms.tms_utime - checkpoint.tms_utime ) << " user, "
                << ( tms.tms_stime - checkpoint.tms_stime ) << " sys" << endl;
            checkpoint = tms;
        }
    }

    if( printout_histograms ) {
        Decomposition::const_iterator tcpIt = tcpSessions.substreamsBegin();

        cout << "== IMAPS conversations ==" << endl;

        cout << tcpSessions.getAttrib("imaps_pkt_count") << " IMAPS packets" << endl;

        while( tcpIt != tcpSessions.substreamsEnd() ) {
            Trace *tcpStream = dynamic_cast<Trace*>(tcpIt->second);
            if( imapsStreamFilter->match( tcpStream->getStreamKey() ) ) {
                cout << "*** " << tcpStream->getStreamKey().toString() 
                    << ", " << tcpStream->getAttrib("count") << " packets" 
                    << ", " << tcpStream->getAttrib("bytecount") << " bytes"
                    << ", " << tcpStream->getAttrib("bytecount_payload") << " bytes payload"
                    << endl;
                cout << "Length histogram: " << tcpStream->getAttrib("length_hist") << endl;
                cout << "Delay histogram: " << tcpStream->getAttrib("delay_hist") << endl;
            }

            ++tcpIt;
        }

        // print out the histogram of IMAPS packet lengths
        cout << tcpSessions.getAttrib("imaps_pkt_count") << " IMAPS packets" << endl;
        cout << "-- IMAPS TCP payload length histogram --" << endl;
        cout << tcpSessions.getAttrib("imaps_length_hist") << endl;
    }

    delete ozyTcpFilter;
    delete ozyFilter;
    delete shakespeareUdpFilter;
    delete shakespeareFilter;
    delete icmpFilter;

    exit(0);
}

---