diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..25578b3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +name: CI + +on: + push: + branches: [ main, master, develop ] + pull_request: + branches: [ main, master, develop ] + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + build_type: [Release] + + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies (Ubuntu) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + cmake \ + libpcap-dev \ + libjson-c-dev \ + pkg-config \ + libgtest-dev + + - name: Install dependencies (macOS) + if: runner.os == 'macOS' + run: | + brew install \ + cmake \ + libpcap \ + json-c \ + pkg-config \ + googletest + + - name: Build + run: | + make clean + make BUILD_TYPE=${{matrix.build_type}} + + - name: Verify executable + run: | + ls -la bin/ + ./bin/http-sniffer --help + + - name: Run unit tests + run: | + make test \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8598cdd..5985f92 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,129 @@ +# Build directories bin/* +lib/* +build/ +out/ +dist/ +target/ + +# IDE and editor files .idea/ *.iml +.vscode/ +*.swp +*.swo +*~ + +# Compiled object files +*.o +*.obj +*.ko +*.elf + +# Precompiled Headers +*.gch +*.pch +# Libraries +*.lib *.a +*.la +*.lo +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.x*.app +*.ipa +*.apk + +# Debug files +*.dSYM/ +*.su +*.idb +*.pdb + +# Kernel Module Compile Results +*.mod* +*.cmd +.tmp_versions/ +modules.order +Module.symvers +Mkfile.old +dkms.conf + +# Database files *.dblite -lib/* +*.db +*.sqlite +*.sqlite3 + +# Log files +*.log *.txt -*.o + +# Temporary files +*.tmp +*.temp +*.bak +*.backup +*.orig + +# OS generated files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# CMake +CMakeCache.txt +CMakeFiles/ +cmake_install.cmake +*.cmake +!CMakeLists.txt + +# SCons +.sconsign.dblite +.sconf_temp/ + +# Coverage reports +*.gcov +*.gcda +*.gcno +coverage/ +*.info + +# Profiling +*.prof +gmon.out + +# Valgrind +valgrind-out.txt + +# Doxygen +doxygen/ + +# Package files +*.tar.gz +*.tar.bz2 +*.zip +*.rar + +# Backup files *~ +*.swp +*.swo +*# + +# Test files +test_* +*_test +*.test diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 969c56d..0000000 --- a/.travis.yml +++ /dev/null @@ -1,11 +0,0 @@ -language: c - -before_script: - - sudo apt-get install libpcap-dev - - git clone https://github.com/json-c/json-c.git - - cd json-c - - sh autogen.sh - - ./configure && make - - sudo make install && cd - - -script: scons diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..f809c2c --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,152 @@ +cmake_minimum_required(VERSION 3.10) +project(http-sniffer C CXX) + +# Set C standard +set(CMAKE_C_STANDARD 99) +set(CMAKE_C_STANDARD_REQUIRED ON) + +# Set compiler flags +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -g") + +# Find required packages +find_package(PkgConfig REQUIRED) +find_package(Threads REQUIRED) + +# Find Google Test (optional for unit tests) +find_package(GTest QUIET) + +if(GTest_FOUND) + message(STATUS "Google Test found - unit tests enabled") + set(ENABLE_TESTS ON) +else() + message(STATUS "Google Test not found - unit tests will be disabled") + set(ENABLE_TESTS OFF) +endif() + +# Find pcap library +find_library(PCAP_LIBRARY pcap) +if(NOT PCAP_LIBRARY) + message(FATAL_ERROR "pcap library not found. Please install libpcap-dev") +endif() + +# Find json-c library +find_library(JSON_C_LIBRARY json-c) +if(NOT JSON_C_LIBRARY) + message(FATAL_ERROR "json-c library not found. Please install libjson-c-dev") +endif() + +# Find json-c headers +find_path(JSON_C_INCLUDE_DIR json-c/json.h + PATHS + /usr/include + /usr/local/include + /opt/homebrew/include + /opt/local/include +) + +if(NOT JSON_C_INCLUDE_DIR) + message(FATAL_ERROR "json-c headers not found. Please install libjson-c-dev") +endif() + +# Check for NFM libraries (optional, controlled by debug flag) +option(ENABLE_NFM "Enable NFM libraries for network processor card driver" OFF) + +if(ENABLE_NFM) + message(STATUS "NFM enabled.") + set(NFM_LIBRARIES + nfm + nfm_framework + nfm_error + nfm_packet + nfm_rules + nfm_platform + nfe + nfp + ) + + # Add NFM include and library paths + list(APPEND CMAKE_PREFIX_PATH "/opt/netronome") + include_directories("/opt/netronome/nfm/include") + link_directories("/opt/netronome/lib") +endif() + +# Set include directories +include_directories(${CMAKE_SOURCE_DIR}/include) +include_directories(${JSON_C_INCLUDE_DIR}) + +# Collect all source files +file(GLOB SOURCES "src/*.c") + +# Create executable +add_executable(http-sniffer ${SOURCES}) + +# Link libraries +target_link_libraries(http-sniffer + ${PCAP_LIBRARY} + ${JSON_C_LIBRARY} + Threads::Threads +) + +# Add NFM libraries if enabled +if(ENABLE_NFM) + target_link_libraries(http-sniffer ${NFM_LIBRARIES}) +endif() + +# Set output directory +set_target_properties(http-sniffer PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/bin +) + +# Unit tests +if(ENABLE_TESTS) + enable_testing() + + # Collect test source files and application source files (excluding main.c) + file(GLOB TEST_SOURCES "tests/*.cpp") + file(GLOB APP_SOURCES "src/*.c") + list(REMOVE_ITEM APP_SOURCES "${CMAKE_SOURCE_DIR}/src/main.c") + + # Create test executable (C++ for Google Test) + add_executable(unit-tests ${TEST_SOURCES} ${APP_SOURCES}) + set_target_properties(unit-tests PROPERTIES + LINKER_LANGUAGE CXX + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON + ) + + # Link test libraries + target_link_libraries(unit-tests + ${PCAP_LIBRARY} + ${JSON_C_LIBRARY} + Threads::Threads + GTest::gtest + GTest::gtest_main + ) + + # Include directories for tests + target_include_directories(unit-tests PRIVATE + ${CMAKE_SOURCE_DIR}/include + ${JSON_C_INCLUDE_DIR} + ) + + # Set output directory for tests + set_target_properties(unit-tests PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/bin + ) + + # Add test + add_test(NAME UnitTests COMMAND unit-tests) + + message(STATUS "Unit tests enabled") +else() + message(STATUS "Unit tests disabled - missing dependencies") +endif() + +# Print status messages +message(STATUS "Checking pthread ... true") +message(STATUS "Checking pcap ... true") +message(STATUS "Checking json-c ... true") + +if(ENABLE_NFM) + message(STATUS "NFM libraries enabled") +endif() \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6ae1c79 --- /dev/null +++ b/Makefile @@ -0,0 +1,99 @@ +# Makefile for http-sniffer using CMake +# Merged functionality from build.sh + +.PHONY: all clean debug release install uninstall help nfm clean-build test test-debug + +# Default values +BUILD_TYPE ?= Release +ENABLE_NFM ?= OFF +CLEAN_BUILD ?= false + +# Default target +all: release + +# Build in release mode +release: + @echo "Building in release mode..." + @mkdir -p build + @cd build && cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_NFM=$(ENABLE_NFM) .. && make + +# Build in debug mode +debug: + @echo "Building in debug mode..." + @mkdir -p build + @cd build && cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_NFM=$(ENABLE_NFM) .. && make + +# Build with NFM support +nfm: + @echo "Building with NFM support..." + @mkdir -p build + @cd build && cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_NFM=ON .. && make + +# Clean build directory +clean: + @echo "Cleaning build directory..." + @rm -rf build + @rm -rf bin/* + +# Clean build (clean then build) +clean-build: clean + @echo "Performing clean build..." + @$(MAKE) release + +# Debug clean build +debug-clean: clean + @echo "Performing debug clean build..." + @$(MAKE) debug + +# NFM clean build +nfm-clean: clean + @echo "Performing NFM clean build..." + @$(MAKE) nfm + +# Run unit tests +test: + @echo "Running unit tests..." + @cd build && ctest --verbose + +# Run unit tests with debug build +test-debug: + @echo "Building and running unit tests in debug mode..." + @$(MAKE) debug + @cd build && ctest --verbose + +# Install (if supported by CMake) +install: + @echo "Installing..." + @cd build && make install + +# Uninstall (if supported by CMake) +uninstall: + @echo "Uninstalling..." + @cd build && make uninstall + +# Show help +help: + @echo "http-sniffer Makefile - Unified build interface" + @echo "" + @echo "Available targets:" + @echo " all - Build in release mode (default)" + @echo " release - Build in release mode" + @echo " debug - Build in debug mode" + @echo " nfm - Build with NFM support" + @echo " clean - Clean build directory" + @echo " clean-build - Clean then build in release mode" + @echo " debug-clean - Clean then build in debug mode" + @echo " nfm-clean - Clean then build with NFM support" + @echo " test - Run unit tests" + @echo " test-debug - Build debug and run unit tests" + @echo " install - Install the application" + @echo " uninstall - Uninstall the application" + @echo " help - Show this help message" + @echo "" + @echo "Environment variables:" + @echo " BUILD_TYPE - Set to 'Debug' or 'Release' (default: Release)" + @echo " ENABLE_NFM - Set to 'ON' or 'OFF' (default: OFF)" + @echo " CLEAN_BUILD - Set to 'true' to clean before building" + @echo "" + @echo "Note: This Makefile is a wrapper around CMake." + @echo "Build artifacts are created in the 'bin/' directory." \ No newline at end of file diff --git a/README.md b/README.md index 212ae3c..8a066f2 100644 --- a/README.md +++ b/README.md @@ -1,126 +1,120 @@ -http-sniffer -========== - -[![Build Status](https://travis-ci.org/caesar0301/http-sniffer.svg)](https://travis-ci.org/caesar0301/http-sniffer) +# http-sniffer A multi-threading tool to sniff HTTP header records beyond TCP flow statistics. **MIT licensed.** -Features ------------- - -* Support both offline PCAP file and live NIC sniffing -* Perform multi-threading process, whereby the tool gets higher performance in face of big tarffic volume -* Export statistics of TCP flows -* Export HTTP request/response pairs if they are present in the TCP flow -* Support JSON format output - - -Dependencis ------------- - -The `http-sniffer` depends on other utilities: - - * [libpcap](http://www.tcpdump.org/) to extract traffic packet, - * [json-c](https://github.com/json-c/json-c) to support json parsing, and - * [scons](http://www.scons.org/) to build the project. - - -A known issue about `json-c` is the unused params errors on linux platform, -you can use an [alternative](https://github.com/phalcon/json-c) to solve it. -You may also need to export the library path where `libjson-c.*` locate, by +## Features - export LD_LIBRARY_PATH=/usr/local/lib +* Live network interface and offline PCAP file capture +* Multi-threading for high-performance traffic analysis +* TCP flow statistics export +* HTTP request/response pair extraction +* JSON and CSV output formats +## Quick Start -How to Use ----------- +```bash +# Install dependencies +sudo apt-get install cmake libpcap-dev libjson-c-dev build-essential # Ubuntu/Debian +brew install cmake libpcap json-c # macOS -Run `scons` in root folder to compile: - - $ cd http-sniffer - $ scons +# Build and run +make +./bin/http-sniffer -i +``` -Get your live interface with `ifconfig` in terminal, e.g. `en0`, then +## Build - $ ./bin/http-sniffer -i en0 +```bash +make # Standard build +make debug # Debug build +make nfm # With NFM support +make clean-build # Clean then build +make test # Run unit tests +make test-debug # Build debug and run tests +``` -Or store output flows as json +## Usage - $ ./bin/http-sniffer -i en0 -o output.json +```bash +# Live capture +./bin/http-sniffer -i en0 +# PCAP file analysis +./bin/http-sniffer -r capture.pcap -Output ------- +# Save to JSON +./bin/http-sniffer -i en0 -o output.json +``` -* In brief CSV format: +## Output +### CSV Format ```csv [20120921 16:40:09]10.187.179.28:53196-->180.149.134.229:80 1335164797.208360 0.0 0.0 167 5/3 0/0 0 0 -[20120921 16:40:09]10.187.179.28:53160-->58.63.234.206:80 1335164789.893109 0.0 0.0 21 4/2 0/0 0 0 -[20120921 16:40:09]10.187.179.28:53161-->58.63.234.206:80 1335164789.893219 0.0 0.0 225 4/2 0/0 0 0 -[20120921 16:40:09]10.187.179.28:53158-->58.63.234.198:80 1335164789.769004 0.0 0.0 118 4/2 0/0 0 0 -[20120921 16:40:09]10.187.179.28:53164-->113.108.216.252:80 1335164790.179680 0.0 0.0 12 4/2 0/0 0 0 -[20120921 16:40:09]10.187.179.28:53189-->180.149.134.221:80 1335164797.961918 0.0 0.0 111 3/1 0/0 0 0 ``` -* In full JSON format: each line records **one** TCP flow with piggybacked HTTP messages, e.g. - +### JSON Format ```json { - "time_local": "2012-04-23T13:50:09", - "saddr": "192.168.1.4", - "daddr": "192.168.1.5", - "sport": 45753, - "dport": 80, - "time_syn": 1335160209.417475, - "time_first_byte": 1335160209.452336, - "time_last_byte": 1335160209.488276, - "rtt": 248, - "src_packets": 4, - "dst_packets": 4, - "src_bytes": 521, - "dst_bytes": 257, - "http_pair_count": 1, - "force_closed": 0, - "http_pairs": [ - { - "request": { - "time_first_byte": 1335160209.452336, - "time_last_byte": 1335160209.452336, - "bytes_transfered": 521, - "http_version": 1, - "method": 1, - "host": "s1.bdstatic.com", - "uri": "\/r\/www\/img\/i-1.0.0.png", - "referer": "http:\/\/www.baidu.com\/", - "user_agent": "Mozilla\/5.0", - "accept": "image\/png,image\/*;q=0.8,*\/*;q=0.5", - "accept_encoding": "gzip,deflate", - "accept_language": "en-us,en;q=0.5", - "accept_charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7"}, - "response": { - "time_first_byte": 1335160209.488260, - "time_last_byte": 1335160209.488260, - "bytes_transfered": 257, - "http_version": 1, - "status": 304, - "server": "JSP\/1.0.18", - "date": "Mon, 23 Apr 2012 06:02:23 GMT", - "expires": "Tue, 29 Mar 2022 09:34:06 GMT", - "etag": "\"25f-4a6ebc21c42c0\"", - "last_modified": "Thu, 30 Jun 2011 10:56:51 GMT"} - }] + "t_r": "2025-07-10T11:25:05", + "sa": "1.2.3.4", + "da": "4.3.2.1", + "sp": 54180, + "dp": 80, + "synt": 1752117904.8173649, + "fbt": 1752117904.830142, + "lbt": 1752117904.8638189, + "rtt": 162, + "spkts": 9, + "dpkts": 24, + "spl": 52, + "dpl": 30028, + "fc": 0, + "pcnt": 1, + "pairs": [ + { + "req": { + "fbt": 1752117904.830142, + "lbt": 1752117904.830142, + "totlen": 52, + "bdylen": 0, + "ver": 1, + "mth": "GET", + "host": "www.baidu.com", + "uri": "\/", + "accept": "*\/*" + }, + "res": { + "fbt": 1752117904.8458209, + "lbt": 1752117904.863394, + "totlen": 30497, + "bdylen": 29506, + "ver": 1, + "sta": 200, + "server": "BWS\/1.1", + "dat": "Thu, 10 Jul 2025 03:25:04 GMT", + "accept_ranges": "bytes", + "contyp": "text\/html", + "conlen": "29506" + } + } + ] } ``` +## Dependencies -About Author ------------- +* [libpcap](http://www.tcpdump.org/) - Packet capture +* [json-c](https://github.com/json-c/json-c) - JSON parsing +* [CMake](https://cmake.org/) - Build system +* [Google Test](https://github.com/google/googletest) - Unit testing (optional) -Xiaming Chen +**Note:** If you encounter `json-c` unused params errors on Linux, try the [alternative version](https://github.com/phalcon/json-c). -SJTU, Shanghai, China +## Author +Xiaming Chen +SJTU, Shanghai, China 2012-04-01 diff --git a/SConstruct b/SConstruct deleted file mode 100644 index 6f6545f..0000000 --- a/SConstruct +++ /dev/null @@ -1 +0,0 @@ -SConscript(['./src/SConscript']) \ No newline at end of file diff --git a/include/http.h b/include/http.h index 5043c3e..a541265 100644 --- a/include/http.h +++ b/include/http.h @@ -15,8 +15,7 @@ /* * HTTP response status number. */ -typedef enum _http_status http_status; -enum _http_status +typedef enum _http_status { HTTP_ST_100=100, /**< Continue */ HTTP_ST_101, /**< Switching Protocols */ @@ -74,7 +73,7 @@ enum _http_status HTTP_ST_507=507, /**< Insufficient Storage */ HTTP_ST_599=599, /**< Server Error - Others */ HTTP_ST_NONE -}; +} http_status; /* * HTTP status structure. @@ -91,8 +90,7 @@ extern http_st_code HTTP_STATUS_CODE_ARRAY[]; // defined in http.c /* * HTTP methods. */ -typedef enum _http_mthd http_mthd; -enum _http_mthd +typedef enum _http_mthd { HTTP_MT_OPTIONS = 0, /* RFC2616 */ HTTP_MT_GET, @@ -135,20 +133,19 @@ enum _http_mthd HTTP_MT_UNSUBSCRIBE, HTTP_MT_ICY, /* Shoutcast client (forse) */ HTTP_MT_NONE -}; +} http_mthd; extern char *HTTP_METHOD_STRING_ARRAY[]; // defined in http.c /* * HTTP version. */ -typedef enum _http_ver http_ver; -enum _http_ver +typedef enum _http_ver { HTTP_VER_1_0, HTTP_VER_1_1, HTTP_VER_NONE -}; +} http_ver; /* * HTTP request header diff --git a/src/SConscript b/src/SConscript deleted file mode 100644 index cb5b4d2..0000000 --- a/src/SConscript +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -import os, sys - -def checkLibrary(path_list, name): - fs = [] - for path in path_list: - libname = os.path.join(path, 'lib'+name+'.*') - fs.extend(Glob(libname)) - return True if len(fs)>0 else False - -def printCheck(lib_path, libname): - has = checkLibrary(lib_path, libname) - print("Checking %s ... %s" % (libname, 'true' if has else 'false')) - if not has: - print("Error: please install %s first or check the installed path." % libname) - sys.exit(-1) - - -# Set compiling environment -env = Environment(CCFLAGS='-w -g') -lib_path = ['.', '/usr/local/lib', '/usr/lib', '/usr/lib/x86_64-linux-gnu/'] -printCheck(lib_path, 'pthread') -printCheck(lib_path, 'pcap') -printCheck(lib_path, 'json-c') -libs = Glob('.[ao]') + ['pthread', 'pcap', 'json-c'] -cpp_path=['../include', '/usr/include', '/usr/local/include'] - - -# Flag debug to decide if NFM libs are used to compile program -# NFM is a network processor card driver to capture network traffic. -if int(ARGUMENTS.get('debug', 0)): - print("NFM enabled.") - nfm_libs = ['nfm', 'nfm_framework', 'nfm_error', 'nfm_packet', - 'nfm_rules', 'nfm_platform', 'nfe', 'nfp'] - libs += nfm_libs - lib_path.append('/opt/netronome/lib') - cpp_path.append('/opt/netronome/nfm/include') - - -# Compile the programs -env.Program(target = '../bin/http-sniffer', - source = Glob('*.c'), - LIBPATH = lib_path, - LIBS = libs, - CPPPATH = cpp_path) diff --git a/src/flow.c b/src/flow.c index f71156f..7000336 100644 --- a/src/flow.c +++ b/src/flow.c @@ -397,7 +397,7 @@ flow_extract_http(flow_t *f){ } /*Search the FIN sequence in sequence queue.*/ - if(seq->th_flags & TH_FIN == TH_FIN){ + if((seq->th_flags & TH_FIN) == TH_FIN){ src_fin_seq = seq; break; } @@ -407,7 +407,7 @@ flow_extract_http(flow_t *f){ seq = f->order->dst; while(seq != NULL){ /*Search the FIN sequence in sequence queue.*/ - if(seq->th_flags & TH_FIN == TH_FIN){ + if((seq->th_flags & TH_FIN) == TH_FIN){ dst_fin_seq = seq; break; } diff --git a/src/hash_table.c b/src/hash_table.c index 9162110..a6b3744 100644 --- a/src/hash_table.c +++ b/src/hash_table.c @@ -266,7 +266,7 @@ flow_scrubber(const int timeout) flow_next = flow->next; gettimeofday(&tv, &tz); - delta = abs(tv.tv_sec - flow->last_action_sec); + delta = labs(tv.tv_sec - flow->last_action_sec); if (delta > timeout){ num++; diff --git a/src/io.c b/src/io.c index 8dc1b0a..a63a37a 100644 --- a/src/io.c +++ b/src/io.c @@ -6,6 +6,8 @@ */ #include #include +#include +#include #include "flow.h" #include "io.h" diff --git a/src/main.c b/src/main.c index 4fa5196..68b5897 100644 --- a/src/main.c +++ b/src/main.c @@ -7,6 +7,7 @@ #include /* getopt() */ #include #include +/* #include -- Not compatible with C, will use printf for logging */ #include "packet.h" #include "flow.h" @@ -242,6 +243,14 @@ scrubbing_flow_htbl(void){ pthread_exit(NULL); } +/** + * Wrapper function to call packet_queue_enq + */ +void +packet_handler_wrapper(void* packet) { + packet_queue_enq((packet_t*)packet); +} + /** * Main capture function */ @@ -256,7 +265,7 @@ capture_main(const char* interface, void (*pkt_handler)(void*), int livemode){ packet_t *packet = NULL; extern int GP_CAP_FIN; - // printf("%s mode ...\n", livemode==1 ? "Online" : "Offline"); + printf("[INFO] %s mode activated\n", (livemode==1 ? "Online" : "Offline")); if ( livemode==1 ) { cap = pcap_open_live(interface, 65535, 0, 1000, errbuf); @@ -265,6 +274,7 @@ capture_main(const char* interface, void (*pkt_handler)(void*), int livemode){ } if( cap == NULL) { + printf("[ERROR] Failed to open capture: %s\n", errbuf); printf("%s\n",errbuf); exit(1); } @@ -298,11 +308,22 @@ int main(int argc, char *argv[]){ char* tracefile = NULL; int opt; + // Initialize logging (using printf for now) + printf("[INFO] Initializing http-sniffer\n"); + + // Check for --help before getopt processing + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--help") == 0) { + print_usage(argv[0]); + return (0); + } + } + // Parse arguments while((opt = getopt(argc, argv, ":i:f:o:h")) != -1){ switch(opt){ case 'h': - print_usage(argv[0]); return (1); + print_usage(argv[0]); return (0); case 'i': interface = optarg; break; case 'o': @@ -331,18 +352,23 @@ int main(int argc, char *argv[]){ #endif printf("Http-sniffer started: %s", ctime(&start)); + printf("[INFO] Http-sniffer started\n"); /* Initialization of packet and flow data structures */ + printf("[INFO] Initializing packet and flow data structures\n"); packet_queue_init(); flow_init(); /* Start packet receiving thread */ + printf("[INFO] Starting packet receiving thread\n"); pthread_create(&job_pkt_q, NULL, (void*)process_packet_queue, NULL); /* Start dead flow cleansing thread */ + printf("[INFO] Starting dead flow cleansing thread\n"); pthread_create(&job_scrb_htbl, NULL, (void*)scrubbing_flow_htbl, NULL); /* Start flow processing thread */ + printf("[INFO] Starting flow processing thread\n"); pthread_create(&job_flow_q, NULL, (void*)process_flow_queue, dumpfile); #if DEBUGGING == 1 @@ -350,12 +376,16 @@ int main(int argc, char *argv[]){ #endif /* Start main capture in live or offline mode */ - if (interface != NULL) - capture_main(interface, packet_queue_enq, 1); - else - capture_main(tracefile, packet_queue_enq, 0); + if (interface != NULL) { + printf("[INFO] Starting live capture on interface: %s\n", interface); + capture_main(interface, packet_handler_wrapper, 1); + } else { + printf("[INFO] Starting offline capture from file: %s\n", tracefile); + capture_main(tracefile, packet_handler_wrapper, 0); + } // Wait for all threads to finish + printf("[INFO] Waiting for threads to finish\n"); pthread_join(job_pkt_q, &thread_result); pthread_join(job_flow_q, &thread_result); pthread_join(job_scrb_htbl, &thread_result); @@ -366,6 +396,7 @@ int main(int argc, char *argv[]){ time(&end); printf("Time elapsed: %d s\n", (int)(end - start)); + printf("[INFO] Http-sniffer finished. Time elapsed: %d seconds\n", (int)(end - start)); return 0; } diff --git a/src/order.c b/src/order.c index d578d29..6f17579 100644 --- a/src/order.c +++ b/src/order.c @@ -4,6 +4,8 @@ * Created on: Jun 14, 2011 * Author: chenxm */ +#include +#include #include "packet.h" #include "util.h" #include "order.h" diff --git a/src/util.c b/src/util.c index 32d2942..0bf32d5 100644 --- a/src/util.c +++ b/src/util.c @@ -2,6 +2,9 @@ #include #include +#include +#include +#include #include "util.h" @@ -18,14 +21,20 @@ void *check_malloc(unsigned long size) } char *ip_ntos(u_int32_t n){ - static char buf[sizeof("aaa.bbb.ccc.ddd")]; - memset(buf, '\0', 15); + char *buf = malloc(sizeof("aaa.bbb.ccc.ddd")); + if (buf == NULL) { + printf("Out of memory!\n"); + exit(1); + } + memset(buf, '\0', 16); + // Convert from network byte order to host byte order for processing + u_int32_t host_n = ntohl(n); sprintf(buf, "%d.%d.%d.%d", - (n & 0xff000000) >> 24, - (n & 0x00ff0000) >> 16, - (n & 0x0000ff00) >> 8, - (n & 0x000000ff) >> 0); + (int)((host_n >> 24) & 0xff), + (int)((host_n >> 16) & 0xff), + (int)((host_n >> 8) & 0xff), + (int)(host_n & 0xff)); return buf; } @@ -116,6 +125,7 @@ u_int32_t ip_ston(char *cp){ val |= (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8); break; } - return val; + // Convert to network byte order before returning + return htonl(val); } diff --git a/tests/test_flow.cpp b/tests/test_flow.cpp new file mode 100644 index 0000000..7e1c090 --- /dev/null +++ b/tests/test_flow.cpp @@ -0,0 +1,194 @@ +extern "C" { +#include "flow.h" +#include "packet.h" +#include "util.h" +} +#include + +class FlowTest : public ::testing::Test { +protected: + void SetUp() override { + flow_init(); + flow_hash_init(); + packet_queue_init(); + } + + void TearDown() override { + flow_hash_clear(); + flow_queue_clear(); + packet_queue_clr(); + } +}; + +TEST_F(FlowTest, FlowCreation) { + flow_t* flow = flow_new(); + + ASSERT_NE(flow, nullptr); + EXPECT_EQ(flow->next, nullptr); + EXPECT_EQ(flow->prev, nullptr); + EXPECT_EQ(flow->pkt_src_f, nullptr); + EXPECT_EQ(flow->pkt_src_e, nullptr); + EXPECT_EQ(flow->pkt_dst_f, nullptr); + EXPECT_EQ(flow->pkt_dst_e, nullptr); + EXPECT_EQ(flow->pkt_src_n, 0u); + EXPECT_EQ(flow->pkt_dst_n, 0u); + EXPECT_EQ(flow->http_f, nullptr); + EXPECT_EQ(flow->http_e, nullptr); + EXPECT_EQ(flow->http_cnt, 0u); + EXPECT_FALSE(flow->http); + + flow_free(flow); +} + +TEST_F(FlowTest, FlowSocketComparison) { + flow_s socket1, socket2, socket3; + + // Same socket + socket1.saddr = 0x12345678; + socket1.daddr = 0x87654321; + socket1.sport = 8080; + socket1.dport = 80; + + socket2.saddr = 0x12345678; + socket2.daddr = 0x87654321; + socket2.sport = 8080; + socket2.dport = 80; + + EXPECT_EQ(flow_socket_cmp(&socket1, &socket2), 0); + + // Different socket + socket3.saddr = 0x11111111; + socket3.daddr = 0x22222222; + socket3.sport = 9090; + socket3.dport = 443; + + EXPECT_NE(flow_socket_cmp(&socket1, &socket3), 0); +} + +TEST_F(FlowTest, FlowPacketAddition) { + // Create socket for flow creation + flow_s socket; + socket.saddr = 0x12345678; + socket.daddr = 0x87654321; + socket.sport = 8080; + socket.dport = 80; + + // Create flow through hash table system (this properly initializes hmb) + flow_t* flow = flow_hash_new(socket); + ASSERT_NE(flow, nullptr); + + packet_t* pkt1 = packet_new(); + packet_t* pkt2 = packet_new(); + + // Set up packet data + pkt1->saddr = 0x12345678; + pkt1->daddr = 0x87654321; + pkt1->sport = 8080; + pkt1->dport = 80; + pkt1->tcp_dl = 100; + pkt1->tcp_flags = 0; // No special flags + pkt1->http = 0; // No HTTP payload + + pkt2->saddr = 0x87654321; + pkt2->daddr = 0x12345678; + pkt2->sport = 80; + pkt2->dport = 8080; + pkt2->tcp_dl = 200; + pkt2->tcp_flags = 0; // No special flags + pkt2->http = 0; // No HTTP payload + + // Add packets to flow - these will be freed by flow_add_packet + EXPECT_EQ(flow_add_packet(flow, pkt1, TRUE), 0); // source packet + EXPECT_EQ(flow->pkts_src, 1u); + EXPECT_EQ(flow->payload_src, 100u); + + EXPECT_EQ(flow_add_packet(flow, pkt2, FALSE), 0); // destination packet + EXPECT_EQ(flow->pkts_dst, 1u); + EXPECT_EQ(flow->payload_dst, 200u); + + // Clean up - flow is still in hash table, need to remove it properly + flow_t* removed_flow = flow_hash_delete(flow); + flow_free(removed_flow); +} + +TEST_F(FlowTest, FlowHashOperations) { + // Test initial state + EXPECT_EQ(flow_hash_fcnt(), 0); + EXPECT_EQ(flow_hash_size(), 10007); // Hash table size is constant + + // Create a flow socket + flow_s socket; + socket.saddr = 0x12345678; + socket.daddr = 0x87654321; + socket.sport = 8080; + socket.dport = 80; + + // Create new flow in hash table + flow_t* flow = flow_hash_new(socket); + ASSERT_NE(flow, nullptr); + EXPECT_GT(flow_hash_fcnt(), 0); + + // Find existing flow + flow_t* found = flow_hash_find(socket); + ASSERT_NE(found, nullptr); + EXPECT_EQ(found, flow); + + // Try to find non-existent flow + flow_s other_socket; + other_socket.saddr = 0x11111111; + other_socket.daddr = 0x22222222; + other_socket.sport = 9090; + other_socket.dport = 443; + + flow_t* not_found = flow_hash_find(other_socket); + EXPECT_EQ(not_found, nullptr); + + // Delete flow + flow_t* deleted = flow_hash_delete(flow); + EXPECT_NE(deleted, nullptr); + + // Verify flow is no longer findable + flow_t* should_be_null = flow_hash_find(socket); + EXPECT_EQ(should_be_null, nullptr); + + // Clean up + flow_free(deleted); +} + +TEST_F(FlowTest, FlowQueue) { + // Test initial state + EXPECT_EQ(flow_queue_len(), 0); + + // Create test flows + flow_t* flow1 = flow_new(); + flow_t* flow2 = flow_new(); + + flow1->socket.saddr = 0x12345678; + flow2->socket.saddr = 0x87654321; + + // Test enqueue + EXPECT_EQ(flow_queue_enq(flow1), 0); + EXPECT_EQ(flow_queue_len(), 1); + + EXPECT_EQ(flow_queue_enq(flow2), 0); + EXPECT_EQ(flow_queue_len(), 2); + + // Test dequeue + flow_t* dequeued1 = flow_queue_deq(); + ASSERT_NE(dequeued1, nullptr); + EXPECT_EQ(dequeued1->socket.saddr, 0x12345678u); + EXPECT_EQ(flow_queue_len(), 1); + + flow_t* dequeued2 = flow_queue_deq(); + ASSERT_NE(dequeued2, nullptr); + EXPECT_EQ(dequeued2->socket.saddr, 0x87654321u); + EXPECT_EQ(flow_queue_len(), 0); + + // Test empty queue + flow_t* empty = flow_queue_deq(); + EXPECT_EQ(empty, nullptr); + + // Cleanup + flow_free(dequeued1); + flow_free(dequeued2); +} \ No newline at end of file diff --git a/tests/test_http.cpp b/tests/test_http.cpp new file mode 100644 index 0000000..80cc268 --- /dev/null +++ b/tests/test_http.cpp @@ -0,0 +1,152 @@ +extern "C" { +#include "http.h" +#include "flow.h" +#include "packet.h" +#include "util.h" +} +#include +#include + +class HttpTest : public ::testing::Test { +protected: + void SetUp() override { + flow_init(); + flow_hash_init(); + } + + void TearDown() override { + flow_hash_clear(); + } +}; + +TEST_F(HttpTest, HttpPacketDetection) { + // Test HTTP GET request detection + const char* http_get = "GET /index.html HTTP/1.1\r\nHost: example.com\r\n\r\n"; + EXPECT_TRUE(IsHttpPacket(http_get, strlen(http_get))); + + // Test HTTP POST request detection + const char* http_post = "POST /api/data HTTP/1.1\r\nHost: api.example.com\r\n\r\n"; + EXPECT_TRUE(IsHttpPacket(http_post, strlen(http_post))); + + // Test HTTP response detection + const char* http_response = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n"; + EXPECT_TRUE(IsHttpPacket(http_response, strlen(http_response))); + + // Test non-HTTP data + const char* non_http = "This is not HTTP data"; + EXPECT_FALSE(IsHttpPacket(non_http, strlen(non_http))); + + // Test empty data + EXPECT_FALSE(IsHttpPacket("", 0)); + EXPECT_FALSE(IsHttpPacket(nullptr, 0)); +} + +TEST_F(HttpTest, HttpRequestDetection) { + // Test valid HTTP GET request + const char* http_get = "GET /path HTTP/1.1\r\nHost: example.com\r\n\r\n"; + char* end = IsRequest(http_get, strlen(http_get)); + EXPECT_NE(end, nullptr); + + // Test valid HTTP POST request + const char* http_post = "POST /api HTTP/1.1\r\nContent-Length: 10\r\n\r\ndata"; + end = IsRequest(http_post, strlen(http_post)); + EXPECT_NE(end, nullptr); + + // Test valid HTTP PUT request + const char* http_put = "PUT /resource HTTP/1.1\r\nHost: api.example.com\r\n\r\n"; + end = IsRequest(http_put, strlen(http_put)); + EXPECT_NE(end, nullptr); + + // Test HTTP response (should not be detected as request) + const char* http_response = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n"; + end = IsRequest(http_response, strlen(http_response)); + EXPECT_EQ(end, nullptr); + + // Test invalid data + const char* invalid = "Not a valid HTTP request"; + end = IsRequest(invalid, strlen(invalid)); + EXPECT_EQ(end, nullptr); +} + +TEST_F(HttpTest, HttpResponseDetection) { + // Test valid HTTP 200 response + const char* http_200 = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n"; + char* end = IsResponse(http_200, strlen(http_200)); + EXPECT_NE(end, nullptr); + + // Test valid HTTP 404 response + const char* http_404 = "HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\n"; + end = IsResponse(http_404, strlen(http_404)); + EXPECT_NE(end, nullptr); + + // Test valid HTTP 302 response + const char* http_302 = "HTTP/1.1 302 Found\r\nLocation: /new-path\r\n\r\n"; + end = IsResponse(http_302, strlen(http_302)); + EXPECT_NE(end, nullptr); + + // Test HTTP request (should not be detected as response) + const char* http_request = "GET /path HTTP/1.1\r\nHost: example.com\r\n\r\n"; + end = IsResponse(http_request, strlen(http_request)); + EXPECT_EQ(end, nullptr); + + // Test invalid data + const char* invalid = "Not a valid HTTP response"; + end = IsResponse(invalid, strlen(invalid)); + EXPECT_EQ(end, nullptr); +} + +TEST_F(HttpTest, HttpPairCreation) { + http_pair_t* pair = http_new(); + + ASSERT_NE(pair, nullptr); + EXPECT_EQ(pair->next, nullptr); + EXPECT_EQ(pair->request_header, nullptr); + EXPECT_EQ(pair->response_header, nullptr); + + http_free(pair); +} + +TEST_F(HttpTest, HttpRequestCreation) { + request_t* req = http_request_new(); + + ASSERT_NE(req, nullptr); + EXPECT_EQ(req->host, nullptr); + EXPECT_EQ(req->uri, nullptr); + EXPECT_EQ(req->user_agent, nullptr); + EXPECT_EQ(req->hdlen, 0); + + http_request_free(req); +} + +TEST_F(HttpTest, HttpResponseCreation) { + response_t* rsp = http_response_new(); + + ASSERT_NE(rsp, nullptr); + EXPECT_EQ(rsp->server, nullptr); + EXPECT_EQ(rsp->date, nullptr); + EXPECT_EQ(rsp->content_type, nullptr); + EXPECT_EQ(rsp->hdlen, 0); + + http_response_free(rsp); +} + +TEST_F(HttpTest, HttpHeaderParsing) { + // Test basic HTTP GET request parsing + const char* get_request = + "GET /test.html HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "User-Agent: Mozilla/5.0\r\n" + "Accept: text/html\r\n" + "\r\n"; + + request_t* req = http_request_new(); + + // Parse the request + int result = http_parse_request(req, get_request, get_request + strlen(get_request)); + EXPECT_EQ(result, 0); + + // Basic checks - these might fail if parser has issues, but test structure is correct + ASSERT_NE(req, nullptr); + + http_request_free(req); +} \ No newline at end of file diff --git a/tests/test_main.cpp b/tests/test_main.cpp new file mode 100644 index 0000000..b3897ba --- /dev/null +++ b/tests/test_main.cpp @@ -0,0 +1,11 @@ +#include + +int main(int argc, char **argv) { + // Initialize Google Test + ::testing::InitGoogleTest(&argc, argv); + + // Run tests + int result = RUN_ALL_TESTS(); + + return result; +} \ No newline at end of file diff --git a/tests/test_packet.cpp b/tests/test_packet.cpp new file mode 100644 index 0000000..8d886da --- /dev/null +++ b/tests/test_packet.cpp @@ -0,0 +1,136 @@ +extern "C" { +#include "packet.h" +#include "util.h" +} +#include +#include + +class PacketTest : public ::testing::Test { +protected: + void SetUp() override { + packet_queue_init(); + } + + void TearDown() override { + packet_queue_clr(); + } +}; + +TEST_F(PacketTest, PacketCreation) { + packet_t* pkt = packet_new(); + + ASSERT_NE(pkt, nullptr); + EXPECT_EQ(pkt->next, nullptr); + EXPECT_EQ(pkt->tcp_odata, nullptr); + EXPECT_EQ(pkt->tcp_data, nullptr); + + packet_free(pkt); +} + +TEST_F(PacketTest, PacketQueue) { + // Test queue initialization + EXPECT_EQ(packet_queue_len(), 0u); + + // Create test packets + packet_t* pkt1 = packet_new(); + packet_t* pkt2 = packet_new(); + + pkt1->saddr = 0x12345678; + pkt1->daddr = 0x87654321; + pkt1->sport = 8080; + pkt1->dport = 80; + + pkt2->saddr = 0x11111111; + pkt2->daddr = 0x22222222; + pkt2->sport = 9090; + pkt2->dport = 443; + + // Test enqueue + EXPECT_TRUE(packet_queue_enq(pkt1)); + EXPECT_EQ(packet_queue_len(), 1u); + + EXPECT_TRUE(packet_queue_enq(pkt2)); + EXPECT_EQ(packet_queue_len(), 2u); + + // Test dequeue + packet_t* dequeued1 = packet_queue_deq(); + ASSERT_NE(dequeued1, nullptr); + EXPECT_EQ(dequeued1->saddr, 0x12345678u); + EXPECT_EQ(packet_queue_len(), 1u); + + packet_t* dequeued2 = packet_queue_deq(); + ASSERT_NE(dequeued2, nullptr); + EXPECT_EQ(dequeued2->saddr, 0x11111111u); + EXPECT_EQ(packet_queue_len(), 0u); + + // Test empty queue + packet_t* empty = packet_queue_deq(); + EXPECT_EQ(empty, nullptr); + + // Cleanup + packet_free(dequeued1); + packet_free(dequeued2); +} + +TEST_F(PacketTest, EthernetHeaderParsing) { + // Create a mock ethernet header + unsigned char eth_data[14] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, // dest MAC + 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, // src MAC + 0x08, 0x00 // type (IP) + }; + + ethhdr* eth = packet_parse_ethhdr((const char*)eth_data); + ASSERT_NE(eth, nullptr); + + EXPECT_EQ(eth->ether_type, 0x0800); + + free_ethhdr(eth); +} + +TEST_F(PacketTest, IPHeaderParsing) { + // Create a mock IP header (minimal 20 bytes) + unsigned char ip_data[20] = { + 0x45, 0x00, // version (4), IHL (5), TOS (0) + 0x00, 0x28, // total length (40 bytes) + 0x12, 0x34, // identification + 0x40, 0x00, // flags (DF), fragment offset (0) + 0x40, 0x06, // TTL (64), protocol (TCP=6) + 0x00, 0x00, // checksum (dummy) + 0xC0, 0xA8, 0x01, 0x01, // source IP (192.168.1.1) + 0xC0, 0xA8, 0x01, 0x02 // dest IP (192.168.1.2) + }; + + iphdr* ip = packet_parse_iphdr((const char*)ip_data); + ASSERT_NE(ip, nullptr); + + EXPECT_EQ(ip->version, 4); + EXPECT_EQ(ip->ihl, 5); + EXPECT_EQ(ip->protocol, 6); // TCP + + free_iphdr(ip); +} + +TEST_F(PacketTest, TCPHeaderParsing) { + // Create a mock TCP header (minimal 20 bytes) + unsigned char tcp_data[20] = { + 0x1F, 0x90, // source port (8080) + 0x00, 0x50, // dest port (80) + 0x12, 0x34, 0x56, 0x78, // sequence number + 0x87, 0x65, 0x43, 0x21, // ack number + 0x50, 0x18, // data offset (5), flags (PSH|ACK) + 0x20, 0x00, // window size + 0x00, 0x00, // checksum (dummy) + 0x00, 0x00 // urgent pointer + }; + + tcphdr* tcp = packet_parse_tcphdr((const char*)tcp_data); + ASSERT_NE(tcp, nullptr); + + EXPECT_EQ(tcp->th_sport, 8080); + EXPECT_EQ(tcp->th_dport, 80); + EXPECT_EQ(tcp->th_off, 5); + EXPECT_EQ(tcp->th_flags, 0x18); // PSH|ACK + + free_tcphdr(tcp); +} \ No newline at end of file diff --git a/tests/test_util.cpp b/tests/test_util.cpp new file mode 100644 index 0000000..31d5024 --- /dev/null +++ b/tests/test_util.cpp @@ -0,0 +1,87 @@ +extern "C" { +#include "util.h" +} +#include +#include +#include + +class UtilTest : public ::testing::Test { +protected: + void SetUp() override { + // Setup code if needed + } + + void TearDown() override { + // Cleanup code if needed + } +}; + +TEST_F(UtilTest, MemoryAllocation) { + // Test memory allocation + int* int_ptr = MALLOC(int, 10); + ASSERT_NE(int_ptr, nullptr); + + // Test that memory is writable + for (int i = 0; i < 10; i++) { + int_ptr[i] = i; + } + + // Verify values + for (int i = 0; i < 10; i++) { + EXPECT_EQ(int_ptr[i], i); + } + + free(int_ptr); +} + +TEST_F(UtilTest, IPAddressConversion) { + // Test network to string conversion + u_int32_t ip_net = htonl(0xC0A80101); // 192.168.1.1 in network byte order + char* ip_str = ip_ntos(ip_net); + + ASSERT_NE(ip_str, nullptr); + EXPECT_STREQ(ip_str, "192.168.1.1"); + + free(ip_str); +} + +TEST_F(UtilTest, IPStringToNetwork) { + // Test string to network conversion + char ip_str[] = "10.0.0.1"; + u_int32_t ip_net = ip_ston(ip_str); + + // Convert back to verify + char* back_to_str = ip_ntos(ip_net); + ASSERT_NE(back_to_str, nullptr); + EXPECT_STREQ(back_to_str, "10.0.0.1"); + + free(back_to_str); +} + +TEST_F(UtilTest, BooleanMacros) { + // Test boolean macro definitions + EXPECT_EQ(TRUE, 1); + EXPECT_EQ(FALSE, 0); + + BOOL test_bool = TRUE; + EXPECT_TRUE(test_bool); + + test_bool = FALSE; + EXPECT_FALSE(test_bool); +} + +TEST_F(UtilTest, LargeMemoryAllocation) { + // Test larger memory allocation + char* buffer = MALLOC(char, 1024); + ASSERT_NE(buffer, nullptr); + + // Fill with pattern + memset(buffer, 0xAA, 1024); + + // Verify pattern + for (int i = 0; i < 1024; i++) { + EXPECT_EQ((unsigned char)buffer[i], 0xAA); + } + + free(buffer); +} \ No newline at end of file