dscanner_linux.cpp

#ifdef __linux__

#include "directory_scanner.hpp"
#include "dscanner_helper.hpp"
#include "config.hpp"

#include <atomic>
#include <cstddef>
#include <queue>
#include <mutex>
#include <sys/syscall.h>
#include <sys/stat.h>
#include <thread>
#include <unistd.h>
#include <fcntl.h>
#include <dirent.h>

#include <cstdint>
#include <string_view>
#include <unordered_set>
#include <utility>
#include <vector>

namespace directory_scanner {
struct linux_dirent64 {
    ino64_t        d_ino;
    off64_t        d_off;
    unsigned short d_reclen;
    unsigned char  d_type;
    char           d_name[];
};

static bool is_dot(const char* name) {
    return (name[0] == '.' && 
    (name[1] == '\0' || 
    (name[1] == '.' && 
    name[2] == '\0')));
}

static inline std::string_view get_extension(const char* name) {
    const char* dot = nullptr;

    for (const char* p =name; *p; ++p) 
        if (*p == '.')
            dot = p;

    if (!dot)
        return {};

    return std::string_view(dot + 1);
}

static bool excluded_extension(const char* name, 
    const std::unordered_set<std::string_view>& excluded_ext) {
    std::string_view ext = get_extension(name);

    if (ext.empty()) return false;

    return excluded_ext.find(ext) != excluded_ext.end();
}

std::vector<FileEntry> scan(const Config& config) {
    std::vector<FileEntry> files;
    files.reserve(16384);

    std::queue<std::string> dir_queue;
    dir_queue.push(config.PATH);

    std::mutex queue_mutex;
    std::mutex file_mutex;

    std::atomic<int> active_workers{0};

    // excluded dirs
    std::unordered_set<std::string_view> excluded;
    for (auto d : config.DEFAULT_EXCLUDED_DIRS)
        excluded.insert(d);
    for (auto& d : config.USER_EXCLUDED_DIRS)
        excluded.insert(d);

    // exlcuded extensions
    std::unordered_set<std::string_view> excluded_ext;
    for (auto e : config.DEFAULT_EXCLUDED_EXTENSIONS)
        excluded_ext.insert(e);
    for (auto& e : config.USER_EXCLUDED_EXTENSIONS)
        excluded_ext.insert(e);

    // gitignore support
    GitIgnore gitignore;
    if (config.USE_GITIGNORE)
        load_gitignore(config.PATH, gitignore);
    for (auto& d : gitignore.ignored_dirs)
        excluded.insert(d);
    for (auto& e : gitignore.ignored_ext)
        excluded_ext.insert(e);

    // size filtering
    uint64_t min_size = config.DEFAULT_MIN_FILE_SIZE * FILE_SIZE;
    uint64_t max_size = config.DEFAULT_MAX_FILE_SIZE * FILE_SIZE;

    bool use_size_filter = (min_size > 0) || (max_size < CAP_FILE_SIZE);

    unsigned workers = config.DEFAULT_WALKER_THREADS;
    std::vector<std::thread> threads;

    auto worker = [&]() {
        constexpr int BUF_SIZE = 262144;
        char buf[BUF_SIZE];

        std::string path;
        path.reserve(4096);

        while (true) {
            std::string dir;
            {
                std::lock_guard<std::mutex> lock(queue_mutex);

                if (dir_queue.empty()) {
                    if (active_workers == 0) return;
                    else continue;
                }

                dir = std::move(dir_queue.front());
                dir_queue.pop();
                active_workers++;
            }
            int fd = open (dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
            if (fd < 0) {
                active_workers--;
                continue;
            }

            path = dir;

            for (;;) {
                int nread = syscall(SYS_getdents64, fd, buf, BUF_SIZE);

                if (nread <= 0) break;

                for (int bpos = 0; bpos < nread;) {
                    linux_dirent64* d = (linux_dirent64*)(buf + bpos);

                    const char* name = d->d_name;

                    if (!is_dot(name)) {
                        size_t old_len = path.size();

                        path.push_back('/');
                        path.append(name);

                        if (d->d_type == DT_DIR) {
                            if (excluded.find(name) == excluded.end()){
                                std::lock_guard<std::mutex> lock(queue_mutex);
                                dir_queue.emplace(path);
                            }
                        } else if (d->d_type == DT_REG) { // TODO: '|| d->d_type==DT_UNKNOWN'
                            if (!excluded_extension(name, excluded_ext)) {
                                struct stat st{};
                                if (fstatat(fd, name, &st, AT_SYMLINK_NOFOLLOW) != 0)
                                    continue;

                                if (!S_ISREG(st.st_mode))
                                    continue;

                                if (st.st_size < 0)
                                    continue;

                                size_t file_size = static_cast<size_t>(st.st_size);

                                if (use_size_filter) {
                                    if (file_size < min_size || file_size > max_size)
                                        continue;
                                }
                                {
                                    std::lock_guard<std::mutex> lock(file_mutex);
                                    files.push_back({path, file_size});
                                }
                            }
                        }
                        path.resize(old_len);
                    }
                    bpos += d->d_reclen;
                }
            }
            close(fd);
            active_workers--;
        }
    };

    for (unsigned i = 0; i < workers; i++)
        threads.emplace_back(worker);

    for (auto& t : threads)
        t.join();

    return files;
}
}
#endif