CrawlerCleanup/parser.cpp at main · BingTea-SearchEngine/CrawlerCleanup · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
#include <string>
#include <filesystem>
#include <iostream>
#include <sstream>
#include <fstream>

using std::cout, std::endl;
namespace fs = std::filesystem;

#include "Parser.hpp"

int main(int argc, char** argv) {
    if (argc < 4) {
        std::cerr << "Expected: " << argv[0] << "<html directory> <out file path> <start number>\n";
        exit(EXIT_FAILURE);
    }

    std::string dirPath = std::string(argv[1]);
    std::string outPath = std::string(argv[2]);
    int startNum = std::stoi(argv[3]);
    std::string line;

    for (const auto& entry : fs::directory_iterator(dirPath)) {
        if (!entry.is_regular_file()) continue;
        std::ifstream f(entry.path());
        std::ostringstream ss;
        ss << f.rdbuf();
        std::string html = std::move(ss.str());
        Parser parser(html);
        for (auto& w : parser.getTitle()) {
            cout << w << " ";
        }
        cout << endl;
        startNum++;
    }
}